8723654472

Committed 17 Apr 2024 02:23PM UTC coverage: 75.676% (-0.3%) from 75.999%

Build # 8723654472

Build Type

Pull #2126

github

Committed by

kubasobon

Commit Message

add add_host_metadata processor for KSPM

Pull Request Pull Request #2126: Add correct host info for cloud provider virtual machines

Run Details

10 of 55 new or added lines in 3 files covered. (18.18%)

2 existing lines in 1 file now uncovered.

7333 of 9690 relevant lines covered (75.68%)

18.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

59.74

/internal/resources/fetching/fetchers/gcp/assets_fetcher.go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package fetchers

import (
        "context"
        "fmt"
        "strings"

        "github.com/elastic/elastic-agent-libs/logp"
        "github.com/huandu/xstrings"

        "github.com/elastic/cloudbeat/internal/resources/fetching"
        "github.com/elastic/cloudbeat/internal/resources/fetching/cycle"
        "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/inventory"
)

type GcpAssetsFetcher struct {
        log        *logp.Logger
        resourceCh chan fetching.ResourceInfo
        provider   inventory.ServiceAPI
}

type GcpAsset struct {
        Type    string
        SubType string

        ExtendedAsset *inventory.ExtendedGcpAsset `json:"asset,omitempty"`
}

// GcpAssetTypes https://cloud.google.com/asset-inventory/docs/supported-asset-types
// map of types to asset types.
// sub-type is derived from asset type by using the first and last segments of the asset type name
// example: gcp-cloudkms-crypto-key
var GcpAssetTypes = map[string][]string{
        fetching.ProjectManagement: {
                inventory.CrmProjectAssetType,
        },
        fetching.KeyManagement: {
                inventory.ApiKeysKeyAssetType,
                inventory.CloudKmsCryptoKeyAssetType,
        },
        fetching.CloudIdentity: {
                inventory.IamServiceAccountAssetType,
                inventory.IamServiceAccountKeyAssetType,
        },
        fetching.CloudDatabase: {
                inventory.BigqueryDatasetAssetType,
                inventory.BigqueryTableAssetType,
                inventory.SqlDatabaseInstanceAssetType,
        },
        fetching.CloudStorage: {
                inventory.StorageBucketAssetType,
                inventory.LogBucketAssetType,
        },
        fetching.CloudCompute: {
                inventory.ComputeInstanceAssetType,
                inventory.ComputeFirewallAssetType,
                inventory.ComputeDiskAssetType,
                inventory.ComputeNetworkAssetType,
                inventory.ComputeBackendServiceAssetType,
                inventory.ComputeSubnetworkAssetType,
        },
        fetching.CloudDns: {
                inventory.DnsManagedZoneAssetType,
        },
        fetching.DataProcessing: {
                inventory.DataprocClusterAssetType,
        },
}

func NewGcpAssetsFetcher(_ context.Context, log *logp.Logger, ch chan fetching.ResourceInfo, provider inventory.ServiceAPI) *GcpAssetsFetcher {
        return &GcpAssetsFetcher{
                log:        log,
                resourceCh: ch,
                provider:   provider,
        }
}

func (f *GcpAssetsFetcher) Fetch(ctx context.Context, cycleMetadata cycle.Metadata) error {
        f.log.Info("Starting GcpAssetsFetcher.Fetch")

        for typeName, assetTypes := range GcpAssetTypes {
                assets, err := f.provider.ListAllAssetTypesByName(ctx, assetTypes)
                if err != nil {
                        f.log.Errorf("Failed to list assets for type %s: %s", typeName, err.Error())
                        continue
                }

                for _, asset := range assets {
                        select {
                        case <-ctx.Done():
                                f.log.Infof("GcpAssetsFetcher.Fetch context err: %s", ctx.Err().Error())
                                return nil
                        case f.resourceCh <- fetching.ResourceInfo{
                                CycleMetadata: cycleMetadata,
                                Resource: &GcpAsset{
                                        Type:          typeName,
                                        SubType:       getGcpSubType(asset.AssetType),
                                        ExtendedAsset: asset,
                                },
                        }:
                        }
                }
        }

        return nil
}

func (f *GcpAssetsFetcher) Stop() {
        f.provider.Close()
}

func (r *GcpAsset) GetData() any {
        return r.ExtendedAsset.Asset
}

func (r *GcpAsset) GetMetadata() (fetching.ResourceMetadata, error) {
        var region string

        if r.ExtendedAsset.Resource != nil {
                region = r.ExtendedAsset.Resource.Location
        }

        return fetching.ResourceMetadata{
                ID:                   r.ExtendedAsset.Name,
                Type:                 r.Type,
                SubType:              r.SubType,
                Name:                 getAssetResourceName(r.ExtendedAsset),
                Region:               region,
                CloudAccountMetadata: *r.ExtendedAsset.CloudAccount,
        }, nil
}

func (r *GcpAsset) GetElasticCommonData() (map[string]any, error) {
        if r.Type == fetching.CloudCompute && r.SubType == inventory.ComputeInstanceAssetType {
                m := map[string]any{}
                data := r.ExtendedAsset.Resource.Data
                if data == nil {
                        return nil, nil
                }
                nameField, ok := data.Fields["name"]
                if ok && nameField.GetStringValue() != "" {
                        m["host.name"] = nameField.GetStringValue()
                }
                hostnameField, ok := data.Fields["hostname"]
                if ok && hostnameField.GetStringValue() != "" {
                        m["host.hostname"] = hostnameField.GetStringValue()
                }
                return m, nil
        }
        return nil, nil
}

// try to retrieve the resource name from the asset data fields (name or displayName), in case it is not set
// get the last part of the asset name (https://cloud.google.com/apis/design/resource_names#resource_id)
func getAssetResourceName(asset *inventory.ExtendedGcpAsset) string {
        if name, exist := asset.GetResource().GetData().GetFields()["displayName"]; exist && name.GetStringValue() != "" {
                return name.GetStringValue()
        }

        if name, exist := asset.GetResource().GetData().GetFields()["name"]; exist && name.GetStringValue() != "" {
                return name.GetStringValue()
        }

        parts := strings.Split(asset.Name, "/")
        return parts[len(parts)-1]
}

func getGcpSubType(assetType string) string {
        dotIndex := strings.Index(assetType, ".")
        slashIndex := strings.Index(assetType, "/")

        prefix := assetType[:dotIndex]
        suffix := assetType[slashIndex+1:]

        return strings.ToLower(fmt.Sprintf("gcp-%s-%s", prefix, xstrings.ToKebabCase(suffix)))
}

1	// Licensed to Elasticsearch B.V. under one or more contributor
2	// license agreements. See the NOTICE file distributed with
3	// this work for additional information regarding copyright
4	// ownership. Elasticsearch B.V. licenses this file to you under
5	// the Apache License, Version 2.0 (the "License"); you may
6	// not use this file except in compliance with the License.
7	// You may obtain a copy of the License at
8	//
9	// http://www.apache.org/licenses/LICENSE-2.0
10	//
11	// Unless required by applicable law or agreed to in writing,
12	// software distributed under the License is distributed on an
13	// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14	// KIND, either express or implied. See the License for the
15	// specific language governing permissions and limitations
16	// under the License.
17
18	package fetchers
19
20	import (
21	"context"
22	"fmt"
23	"strings"
24
25	"github.com/elastic/elastic-agent-libs/logp"
26	"github.com/huandu/xstrings"
27
28	"github.com/elastic/cloudbeat/internal/resources/fetching"
29	"github.com/elastic/cloudbeat/internal/resources/fetching/cycle"
30	"github.com/elastic/cloudbeat/internal/resources/providers/gcplib/inventory"
31	)
32
33	type GcpAssetsFetcher struct {
34	log *logp.Logger
35	resourceCh chan fetching.ResourceInfo
36	provider inventory.ServiceAPI
37	}
38
39	type GcpAsset struct {
40	Type string
41	SubType string
42
43	ExtendedAsset *inventory.ExtendedGcpAsset `json:"asset,omitempty"`
44	}
45
46	// GcpAssetTypes https://cloud.google.com/asset-inventory/docs/supported-asset-types
47	// map of types to asset types.
48	// sub-type is derived from asset type by using the first and last segments of the asset type name
49	// example: gcp-cloudkms-crypto-key
50	var GcpAssetTypes = map[string][]string{
51	fetching.ProjectManagement: {
52	inventory.CrmProjectAssetType,
53	},
54	fetching.KeyManagement: {
55	inventory.ApiKeysKeyAssetType,
56	inventory.CloudKmsCryptoKeyAssetType,
57	},
58	fetching.CloudIdentity: {
59	inventory.IamServiceAccountAssetType,
60	inventory.IamServiceAccountKeyAssetType,
61	},
62	fetching.CloudDatabase: {
63	inventory.BigqueryDatasetAssetType,
64	inventory.BigqueryTableAssetType,
65	inventory.SqlDatabaseInstanceAssetType,
66	},
67	fetching.CloudStorage: {
68	inventory.StorageBucketAssetType,
69	inventory.LogBucketAssetType,
70	},
71	fetching.CloudCompute: {
72	inventory.ComputeInstanceAssetType,
73	inventory.ComputeFirewallAssetType,
74	inventory.ComputeDiskAssetType,
75	inventory.ComputeNetworkAssetType,
76	inventory.ComputeBackendServiceAssetType,
77	inventory.ComputeSubnetworkAssetType,
78	},
79	fetching.CloudDns: {
80	inventory.DnsManagedZoneAssetType,
81	},
82	fetching.DataProcessing: {
83	inventory.DataprocClusterAssetType,
84	},
85	}
86
87	func NewGcpAssetsFetcher(_ context.Context, log logp.Logger, ch chan fetching.ResourceInfo, provider inventory.ServiceAPI) GcpAssetsFetcher {	2✔
88	return &GcpAssetsFetcher{	2✔
89	log: log,	2✔
90	resourceCh: ch,	2✔
91	provider: provider,	2✔
92	}	2✔
93	}	2✔
94
95	func (f *GcpAssetsFetcher) Fetch(ctx context.Context, cycleMetadata cycle.Metadata) error {	1✔
96	f.log.Info("Starting GcpAssetsFetcher.Fetch")	1✔
97		1✔
98	for typeName, assetTypes := range GcpAssetTypes {	9✔
99	assets, err := f.provider.ListAllAssetTypesByName(ctx, assetTypes)	8✔
100	if err != nil {	8✔
101	f.log.Errorf("Failed to list assets for type %s: %s", typeName, err.Error())	×
102	continue	×
103	}
104
105	for _, asset := range assets {	16✔
106	select {	8✔
107	case <-ctx.Done():	×
108	f.log.Infof("GcpAssetsFetcher.Fetch context err: %s", ctx.Err().Error())	×
109	return nil	×
110	case f.resourceCh <- fetching.ResourceInfo{
111	CycleMetadata: cycleMetadata,
112	Resource: &GcpAsset{
113	Type: typeName,
114	SubType: getGcpSubType(asset.AssetType),
115	ExtendedAsset: asset,
116	},
117	}:	8✔
118	}
119	}
120	}
121
122	return nil	1✔
123	}
124
125	func (f *GcpAssetsFetcher) Stop() {	2✔
126	f.provider.Close()	2✔
127	}	2✔
128
129	func (r *GcpAsset) GetData() any {	×
130	return r.ExtendedAsset.Asset	×
131	}	×
132
133	func (r *GcpAsset) GetMetadata() (fetching.ResourceMetadata, error) {	8✔
134	var region string	8✔
135		8✔
136	if r.ExtendedAsset.Resource != nil {	8✔
137	region = r.ExtendedAsset.Resource.Location	×
138	}	×
139
140	return fetching.ResourceMetadata{	8✔
141	ID: r.ExtendedAsset.Name,	8✔
142	Type: r.Type,	8✔
143	SubType: r.SubType,	8✔
144	Name: getAssetResourceName(r.ExtendedAsset),	8✔
145	Region: region,	8✔
146	CloudAccountMetadata: *r.ExtendedAsset.CloudAccount,	8✔
147	}, nil	8✔
148	}
149
150	func (r *GcpAsset) GetElasticCommonData() (map[string]any, error) {	×
NEW 151	if r.Type == fetching.CloudCompute && r.SubType == inventory.ComputeInstanceAssetType {	×
NEW 152	m := map[string]any{}	×
NEW 153	data := r.ExtendedAsset.Resource.Data	×
NEW 154	if data == nil {	×
NEW 155	return nil, nil	×
NEW 156	}	×
NEW 157	nameField, ok := data.Fields["name"]	×
NEW 158	if ok && nameField.GetStringValue() != "" {	×
NEW 159	m["host.name"] = nameField.GetStringValue()	×
NEW 160	}	×
NEW 161	hostnameField, ok := data.Fields["hostname"]	×
NEW 162	if ok && hostnameField.GetStringValue() != "" {	×
NEW 163	m["host.hostname"] = hostnameField.GetStringValue()	×
NEW 164	}	×
NEW 165	return m, nil	×
166	}
167	return nil, nil	×
168	}
169
170	// try to retrieve the resource name from the asset data fields (name or displayName), in case it is not set
171	// get the last part of the asset name (https://cloud.google.com/apis/design/resource_names#resource_id)
172	func getAssetResourceName(asset *inventory.ExtendedGcpAsset) string {	8✔
173	if name, exist := asset.GetResource().GetData().GetFields()["displayName"]; exist && name.GetStringValue() != "" {	8✔
UNCOV 174	return name.GetStringValue()	×
UNCOV 175	}	×
176
177	if name, exist := asset.GetResource().GetData().GetFields()["name"]; exist && name.GetStringValue() != "" {	8✔
178	return name.GetStringValue()	×
179	}	×
180
181	parts := strings.Split(asset.Name, "/")	8✔
182	return parts[len(parts)-1]	8✔
183	}
184
185	func getGcpSubType(assetType string) string {	8✔
186	dotIndex := strings.Index(assetType, ".")	8✔
187	slashIndex := strings.Index(assetType, "/")	8✔
188		8✔
189	prefix := assetType[:dotIndex]	8✔
190	suffix := assetType[slashIndex+1:]	8✔
191		8✔
192	return strings.ToLower(fmt.Sprintf("gcp-%s-%s", prefix, xstrings.ToKebabCase(suffix)))	8✔
193	}	8✔

elastic / cloudbeat / 8723654472

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous