14900461121

Committed 08 May 2025 06:49AM UTC coverage: 76.227% (+0.1%) from 76.125%

Build # 14900461121

Build Type

Pull #3250

github

Committed by

moukoublen

Commit Message

log errors on env vars parsing

Pull Request Pull Request #3250: Increase gcp ListAssets timeout and page size

Run Details

57 of 67 new or added lines in 5 files covered. (85.07%)

15 existing lines in 2 files now uncovered.

9209 of 12081 relevant lines covered (76.23%)

16.52 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.44

/internal/config/config.go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

// Config is put into a different package to prevent cyclic imports in case
// it is needed in several locations

package config

import (
        "fmt"
        "os"
        "path/filepath"
        "strconv"
        "time"

        "github.com/elastic/beats/v7/libbeat/processors"
        "github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
        "github.com/elastic/elastic-agent-libs/config"

        "github.com/elastic/cloudbeat/internal/infra/clog"
        "github.com/elastic/cloudbeat/internal/launcher"
)

const (
        DefaultNamespace                = "default"
        VulnerabilityType               = "vuln_mgmt"
        AssetInventoryType              = "asset_inventory"
        defaultFindingsIndexPrefix      = "logs-cloud_security_posture.findings"
        defaultVulnerabilityIndexPrefix = "logs-cloud_security_posture.vulnerabilities"
)

type Fetcher struct {
        Name string `config:"name"` // Name of the fetcher
}

type Config struct {
        Benchmark              string                  `config:"config.v1.benchmark"`
        Type                   string                  `config:"config.v1.type"`
        Deployment             string                  `config:"config.v1.deployment"`
        AssetInventoryProvider string                  `config:"config.v1.asset_inventory_provider"`
        CloudConfig            CloudConfig             `config:"config.v1"`
        KubeConfig             string                  `config:"kube_config"`
        Period                 time.Duration           `config:"period"`
        Processors             processors.PluginConfig `config:"processors"`
        BundlePath             string                  `config:"bundle_path"`
        PackagePolicyId        string                  `config:"package_policy_id"`
        PackagePolicyRevision  int                     `config:"revision"`
        Index                  string                  `config:"index"`
}

type CloudConfig struct {
        Aws   AwsConfig   `config:"aws"`
        Gcp   GcpConfig   `config:"gcp"`
        Azure AzureConfig `config:"azure"`
}

type AwsConfig struct {
        Cred                  aws.ConfigAWS `config:"credentials"`
        AccountType           string        `config:"account_type"`
        CloudConnectors       bool          `config:"supports_cloud_connectors"`
        CloudConnectorsConfig CloudConnectorsConfig
}

type GcpConfig struct {
        // empty for OrganizationAccount
        ProjectId string `config:"project_id"`

        // empty for SingleAccount
        OrganizationId string `config:"organization_id"`

        // SingleAccount or OrganizationAccount
        AccountType string `config:"account_type"`

        GcpCallOpt GcpCallOpt `config:"call_options"`

        GcpClientOpt `config:"credentials"`
}

type GcpClientOpt struct {
        CredentialsJSON     string `config:"credentials_json"`
        CredentialsFilePath string `config:"credentials_file_path"`
}

type GcpCallOpt struct {
        ListAssetsTimeout  time.Duration `config:"list_assets_timeout"`
        ListAssetsPageSize int32         `config:"list_assets_page_size"`
}

type AzureConfig struct {
        Credentials AzureClientOpt `config:"credentials"`
        // SingleAccount or OrganizationAccount
        AccountType string `config:"account_type"`
}

type AzureClientOpt struct {
        ClientCredentialsType     string `config:"type"`
        ClientID                  string `config:"client_id"`
        TenantID                  string `config:"tenant_id"`
        ClientSecret              string `config:"client_secret"`
        ClientUsername            string `config:"client_username"`
        ClientPassword            string `config:"client_password"`
        ClientCertificatePath     string `config:"client_certificate_path"`
        ClientCertificatePassword string `config:"client_certificate_password"`
}

const (
        AzureClientCredentialsTypeARMTemplate     = "arm_template"
        AzureClientCredentialsTypeManagedIdentity = "managed_identity"
        AzureClientCredentialsTypeManual          = "manual"
        AzureClientCredentialsTypeSecret          = "service_principal_with_client_secret"
        AzureClientCredentialsTypeCertificate     = "service_principal_with_client_certificate"
)

const (
        SingleAccount       = "single-account"
        OrganizationAccount = "organization-account"
)

// Datastream returns the name of a Data Stream to publish Cloudbeat events to.
func (c *Config) Datastream() string {
        if c.Index != "" {
                return c.Index
        }
        if c.Type == VulnerabilityType {
                return defaultVulnerabilityIndexPrefix + "-" + DefaultNamespace
        }
        return defaultFindingsIndexPrefix + "-" + DefaultNamespace
}

func New(cfg *config.C) (*Config, error) {
        c, err := defaultConfig()
        if err != nil {
                return nil, err
        }

        if err := cfg.Unpack(&c); err != nil {
                return nil, err
        }

        if c.Benchmark != "" {
                if !isSupportedBenchmark(c.Benchmark) {
                        return c, launcher.NewUnhealthyError(fmt.Sprintf("benchmark '%s' is not supported", c.Benchmark))
                }
        }

        // apply env var overwrites
        overwritesFromEnvVars(c)

        switch c.CloudConfig.Aws.AccountType {
        case "":
        case SingleAccount:
        case OrganizationAccount:
        default:
                return nil, launcher.NewUnhealthyError(fmt.Sprintf(
                        "aws.account_type '%s' is not supported",
                        c.CloudConfig.Aws.AccountType,
                ))
        }

        switch c.CloudConfig.Azure.AccountType {
        case "":
        case SingleAccount:
        case OrganizationAccount:
        default:
                return nil, launcher.NewUnhealthyError(fmt.Sprintf(
                        "azure.account_type '%s' is not supported",
                        c.CloudConfig.Azure.AccountType,
                ))
        }

        if c.CloudConfig.Aws.CloudConnectors {
                c.CloudConfig.Aws.CloudConnectorsConfig = newCloudConnectorsConfig()
        }

        return c, nil
}

func defaultConfig() (*Config, error) {
        ret := &Config{
                Period: 4 * time.Hour,
                CloudConfig: CloudConfig{
                        Gcp: defaultGCPConfig(),
                },
        }

        bundle, err := getBundlePath()
        if err != nil {
                return nil, err
        }

        ret.BundlePath = bundle
        return ret, nil
}

func defaultGCPConfig() GcpConfig {
        return GcpConfig{
                GcpCallOpt: GcpCallOpt{
                        // default value from sdk is 1m; we use 4m to exceed quota window.
                        // https://github.com/googleapis/google-cloud-go/blob/952cd7fd419af9eb74f5d30a111ae936094b0645/asset/apiv1/asset_client.go#L96
                        ListAssetsTimeout: 4 * time.Minute,

                        // default value from sdk is 100; we use 200
                        // https://github.com/googleapis/google-cloud-go/blob/a6c85f6387ee6aa291e786c882637fb03f3302f4/asset/apiv1/assetpb/asset_service.pb.go#L767-L769
                        ListAssetsPageSize: 200,
                },
        }
}

func getBundlePath() (string, error) {
        // The bundle resides on the same location as the executable
        ex, err := os.Executable()
        if err != nil {
                return "", err
        }
        return filepath.Join(filepath.Dir(ex), "bundle.tar.gz"), nil
}

func isSupportedBenchmark(benchmark string) bool {
        for _, s := range SupportedCIS {
                if benchmark == s {
                        return true
                }
        }
        return false
}

// Cloud Connectors roles and resource id must be provided by the system (controller)
// and not user input (package policy) for security reasons.

const (
        CloudConnectorsLocalRoleEnvVar  = "CLOUD_CONNECTORS_LOCAL_ROLE"
        CloudConnectorsGlobalRoleEnvVar = "CLOUD_CONNECTORS_GLOBAL_ROLE"
        CloudResourceIDEnvVar           = "CLOUD_RESOURCE_ID"
)

type CloudConnectorsConfig struct {
        LocalRoleARN  string
        GlobalRoleARN string
        ResourceID    string
}

func newCloudConnectorsConfig() CloudConnectorsConfig {
        return CloudConnectorsConfig{
                LocalRoleARN:  os.Getenv(CloudConnectorsLocalRoleEnvVar),
                GlobalRoleARN: os.Getenv(CloudConnectorsGlobalRoleEnvVar),
                ResourceID:    os.Getenv(CloudResourceIDEnvVar),
        }
}

const (
        CloudbeatGCPListAssetPageSizeEnvVar = "CLOUDBEAT_GCP_LIST_ASSETS_PAGE_SIZE"
        CloudbeatGCPListAssetTimeoutEnvVar  = "CLOUDBEAT_GCP_LIST_ASSETS_TIMEOUT"
)

func overwritesFromEnvVars(c *Config) {
        log := clog.NewLogger("config")
        logErr := func(name string, value string, err error) {
                log.Errorf("error trying to parse variable %s with value %s: %s", name, value, err.Error())
        }

        if v, exists := os.LookupEnv(CloudbeatGCPListAssetPageSizeEnvVar); exists {
                if i, err := strconv.ParseInt(v, 10, 32); err == nil {
                        c.CloudConfig.Gcp.GcpCallOpt.ListAssetsPageSize = int32(i)
                } else {
                        logErr(CloudbeatGCPListAssetPageSizeEnvVar, v, err)
                }
        }

        if v, exists := os.LookupEnv(CloudbeatGCPListAssetTimeoutEnvVar); exists {
                if d, err := time.ParseDuration(v); err == nil {
                        c.CloudConfig.Gcp.GcpCallOpt.ListAssetsTimeout = d
                } else {
                        logErr(CloudbeatGCPListAssetPageSizeEnvVar, v, err)
                }
        }
}

1	// Licensed to Elasticsearch B.V. under one or more contributor
2	// license agreements. See the NOTICE file distributed with
3	// this work for additional information regarding copyright
4	// ownership. Elasticsearch B.V. licenses this file to you under
5	// the Apache License, Version 2.0 (the "License"); you may
6	// not use this file except in compliance with the License.
7	// You may obtain a copy of the License at
8	//
9	// http://www.apache.org/licenses/LICENSE-2.0
10	//
11	// Unless required by applicable law or agreed to in writing,
12	// software distributed under the License is distributed on an
13	// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14	// KIND, either express or implied. See the License for the
15	// specific language governing permissions and limitations
16	// under the License.
17
18	// Config is put into a different package to prevent cyclic imports in case
19	// it is needed in several locations
20
21	package config
22
23	import (
24	"fmt"
25	"os"
26	"path/filepath"
27	"strconv"
28	"time"
29
30	"github.com/elastic/beats/v7/libbeat/processors"
31	"github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
32	"github.com/elastic/elastic-agent-libs/config"
33
34	"github.com/elastic/cloudbeat/internal/infra/clog"
35	"github.com/elastic/cloudbeat/internal/launcher"
36	)
37
38	const (
39	DefaultNamespace = "default"
40	VulnerabilityType = "vuln_mgmt"
41	AssetInventoryType = "asset_inventory"
42	defaultFindingsIndexPrefix = "logs-cloud_security_posture.findings"
43	defaultVulnerabilityIndexPrefix = "logs-cloud_security_posture.vulnerabilities"
44	)
45
46	type Fetcher struct {
47	Name string `config:"name"` // Name of the fetcher
48	}
49
50	type Config struct {
51	Benchmark string `config:"config.v1.benchmark"`
52	Type string `config:"config.v1.type"`
53	Deployment string `config:"config.v1.deployment"`
54	AssetInventoryProvider string `config:"config.v1.asset_inventory_provider"`
55	CloudConfig CloudConfig `config:"config.v1"`
56	KubeConfig string `config:"kube_config"`
57	Period time.Duration `config:"period"`
58	Processors processors.PluginConfig `config:"processors"`
59	BundlePath string `config:"bundle_path"`
60	PackagePolicyId string `config:"package_policy_id"`
61	PackagePolicyRevision int `config:"revision"`
62	Index string `config:"index"`
63	}
64
65	type CloudConfig struct {
66	Aws AwsConfig `config:"aws"`
67	Gcp GcpConfig `config:"gcp"`
68	Azure AzureConfig `config:"azure"`
69	}
70
71	type AwsConfig struct {
72	Cred aws.ConfigAWS `config:"credentials"`
73	AccountType string `config:"account_type"`
74	CloudConnectors bool `config:"supports_cloud_connectors"`
75	CloudConnectorsConfig CloudConnectorsConfig
76	}
77
78	type GcpConfig struct {
79	// empty for OrganizationAccount
80	ProjectId string `config:"project_id"`
81
82	// empty for SingleAccount
83	OrganizationId string `config:"organization_id"`
84
85	// SingleAccount or OrganizationAccount
86	AccountType string `config:"account_type"`
87
88	GcpCallOpt GcpCallOpt `config:"call_options"`
89
90	GcpClientOpt `config:"credentials"`
91	}
92
93	type GcpClientOpt struct {
94	CredentialsJSON string `config:"credentials_json"`
95	CredentialsFilePath string `config:"credentials_file_path"`
96	}
97
98	type GcpCallOpt struct {
99	ListAssetsTimeout time.Duration `config:"list_assets_timeout"`
100	ListAssetsPageSize int32 `config:"list_assets_page_size"`
101	}
102
103	type AzureConfig struct {
104	Credentials AzureClientOpt `config:"credentials"`
105	// SingleAccount or OrganizationAccount
106	AccountType string `config:"account_type"`
107	}
108
109	type AzureClientOpt struct {
110	ClientCredentialsType string `config:"type"`
111	ClientID string `config:"client_id"`
112	TenantID string `config:"tenant_id"`
113	ClientSecret string `config:"client_secret"`
114	ClientUsername string `config:"client_username"`
115	ClientPassword string `config:"client_password"`
116	ClientCertificatePath string `config:"client_certificate_path"`
117	ClientCertificatePassword string `config:"client_certificate_password"`
118	}
119
120	const (
121	AzureClientCredentialsTypeARMTemplate = "arm_template"
122	AzureClientCredentialsTypeManagedIdentity = "managed_identity"
123	AzureClientCredentialsTypeManual = "manual"
124	AzureClientCredentialsTypeSecret = "service_principal_with_client_secret"
125	AzureClientCredentialsTypeCertificate = "service_principal_with_client_certificate"
126	)
127
128	const (
129	SingleAccount = "single-account"
130	OrganizationAccount = "organization-account"
131	)
132
133	// Datastream returns the name of a Data Stream to publish Cloudbeat events to.
134	func (c *Config) Datastream() string {	7✔
135	if c.Index != "" {	7✔
UNCOV 136	return c.Index	×
137	}	×
138	if c.Type == VulnerabilityType {	7✔
UNCOV 139	return defaultVulnerabilityIndexPrefix + "-" + DefaultNamespace	×
140	}	×
141	return defaultFindingsIndexPrefix + "-" + DefaultNamespace	7✔
142	}
143
144	func New(cfg config.C) (Config, error) {	22✔
145	c, err := defaultConfig()	22✔
146	if err != nil {	22✔
UNCOV 147	return nil, err	×
148	}	×
149
150	if err := cfg.Unpack(&c); err != nil {	22✔
UNCOV 151	return nil, err	×
152	}	×
153
154	if c.Benchmark != "" {	35✔
155	if !isSupportedBenchmark(c.Benchmark) {	13✔
UNCOV 156	return c, launcher.NewUnhealthyError(fmt.Sprintf("benchmark '%s' is not supported", c.Benchmark))	×
157	}	×
158	}
159
160	// apply env var overwrites
161	overwritesFromEnvVars(c)	22✔
162		22✔
163	switch c.CloudConfig.Aws.AccountType {	22✔
164	case "":	19✔
165	case SingleAccount:	2✔
166	case OrganizationAccount:	1✔
UNCOV 167	default:	×
168	return nil, launcher.NewUnhealthyError(fmt.Sprintf(	×
169	"aws.account_type '%s' is not supported",	×
170	c.CloudConfig.Aws.AccountType,	×
171	))	×
172	}
173
174	switch c.CloudConfig.Azure.AccountType {	22✔
175	case "":	22✔
UNCOV 176	case SingleAccount:	×
177	case OrganizationAccount:	×
178	default:	×
179	return nil, launcher.NewUnhealthyError(fmt.Sprintf(	×
180	"azure.account_type '%s' is not supported",	×
181	c.CloudConfig.Azure.AccountType,	×
182	))	×
183	}
184
185	if c.CloudConfig.Aws.CloudConnectors {	25✔
186	c.CloudConfig.Aws.CloudConnectorsConfig = newCloudConnectorsConfig()	3✔
187	}	3✔
188
189	return c, nil	22✔
190	}
191
192	func defaultConfig() (*Config, error) {	22✔
193	ret := &Config{	22✔
194	Period: 4 * time.Hour,	22✔
195	CloudConfig: CloudConfig{	22✔
196	Gcp: defaultGCPConfig(),	22✔
197	},	22✔
198	}	22✔
199		22✔
200	bundle, err := getBundlePath()	22✔
201	if err != nil {	22✔
UNCOV 202	return nil, err	×
203	}	×
204
205	ret.BundlePath = bundle	22✔
206	return ret, nil	22✔
207	}
208
209	func defaultGCPConfig() GcpConfig {	28✔
210	return GcpConfig{	28✔
211	GcpCallOpt: GcpCallOpt{	28✔
212	// default value from sdk is 1m; we use 4m to exceed quota window.	28✔
213	// https://github.com/googleapis/google-cloud-go/blob/952cd7fd419af9eb74f5d30a111ae936094b0645/asset/apiv1/asset_client.go#L96	28✔
214	ListAssetsTimeout: 4 * time.Minute,	28✔
215		28✔
216	// default value from sdk is 100; we use 200	28✔
217	// https://github.com/googleapis/google-cloud-go/blob/a6c85f6387ee6aa291e786c882637fb03f3302f4/asset/apiv1/assetpb/asset_service.pb.go#L767-L769	28✔
218	ListAssetsPageSize: 200,	28✔
219	},	28✔
220	}	28✔
221	}	28✔
222
223	func getBundlePath() (string, error) {	22✔
224	// The bundle resides on the same location as the executable	22✔
225	ex, err := os.Executable()	22✔
226	if err != nil {	22✔
UNCOV 227	return "", err	×
228	}	×
229	return filepath.Join(filepath.Dir(ex), "bundle.tar.gz"), nil	22✔
230	}
231
232	func isSupportedBenchmark(benchmark string) bool {	13✔
233	for _, s := range SupportedCIS {	52✔
234	if benchmark == s {	52✔
235	return true	13✔
236	}	13✔
237	}
UNCOV 238	return false	×
239	}
240
241	// Cloud Connectors roles and resource id must be provided by the system (controller)
242	// and not user input (package policy) for security reasons.
243
244	const (
245	CloudConnectorsLocalRoleEnvVar = "CLOUD_CONNECTORS_LOCAL_ROLE"
246	CloudConnectorsGlobalRoleEnvVar = "CLOUD_CONNECTORS_GLOBAL_ROLE"
247	CloudResourceIDEnvVar = "CLOUD_RESOURCE_ID"
248	)
249
250	type CloudConnectorsConfig struct {
251	LocalRoleARN string
252	GlobalRoleARN string
253	ResourceID string
254	}
255
256	func newCloudConnectorsConfig() CloudConnectorsConfig {	3✔
257	return CloudConnectorsConfig{	3✔
258	LocalRoleARN: os.Getenv(CloudConnectorsLocalRoleEnvVar),	3✔
259	GlobalRoleARN: os.Getenv(CloudConnectorsGlobalRoleEnvVar),	3✔
260	ResourceID: os.Getenv(CloudResourceIDEnvVar),	3✔
261	}	3✔
262	}	3✔
263
264	const (
265	CloudbeatGCPListAssetPageSizeEnvVar = "CLOUDBEAT_GCP_LIST_ASSETS_PAGE_SIZE"
266	CloudbeatGCPListAssetTimeoutEnvVar = "CLOUDBEAT_GCP_LIST_ASSETS_TIMEOUT"
267	)
268
269	func overwritesFromEnvVars(c *Config) {	22✔
270	log := clog.NewLogger("config")	22✔
271	logErr := func(name string, value string, err error) {	22✔
NEW 272	log.Errorf("error trying to parse variable %s with value %s: %s", name, value, err.Error())	×
NEW 273	}	×
274
275	if v, exists := os.LookupEnv(CloudbeatGCPListAssetPageSizeEnvVar); exists {	23✔
276	if i, err := strconv.ParseInt(v, 10, 32); err == nil {	2✔
277	c.CloudConfig.Gcp.GcpCallOpt.ListAssetsPageSize = int32(i)	1✔
278	} else {	1✔
NEW 279	logErr(CloudbeatGCPListAssetPageSizeEnvVar, v, err)	×
NEW 280	}	×
281	}
282
283	if v, exists := os.LookupEnv(CloudbeatGCPListAssetTimeoutEnvVar); exists {	23✔
284	if d, err := time.ParseDuration(v); err == nil {	2✔
285	c.CloudConfig.Gcp.GcpCallOpt.ListAssetsTimeout = d	1✔
286	} else {	1✔
NEW 287	logErr(CloudbeatGCPListAssetPageSizeEnvVar, v, err)	×
NEW 288	}	×
289	}
290	}

elastic / cloudbeat / 14900461121

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous