01875e2f-959c-4c4d-87af-1d7805759bcc

Committed 08 Apr 2023 12:26AM UTC coverage: 57.178% (+0.1%) from 57.072%

Build # 01875e2f-959c-4c4d-87af-1d7805759bcc

Build Type

buildkite

Committed by Steven L

Commit Message

bad cleanup -> good cleanup

Pull Request Pull Request #5197: Demonstrate a way to get rid of the cadence-idl repo

Run Details

85396 of 149351 relevant lines covered (57.18%)

2283.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.58

/common/peerprovider/ringpopprovider/config.go

// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package ringpopprovider

import (
        "context"
        "errors"
        "fmt"
        "net"
        "strings"
        "time"

        "github.com/uber/ringpop-go/discovery"
        "github.com/uber/ringpop-go/discovery/jsonfile"
        "github.com/uber/ringpop-go/discovery/statichosts"

        "github.com/uber/cadence/common/log"
        "github.com/uber/cadence/common/log/tag"
)

// BootstrapMode is an enum type for ringpop bootstrap mode
type BootstrapMode int

const (
        // BootstrapModeNone represents a bootstrap mode set to nothing or invalid
        BootstrapModeNone BootstrapMode = iota
        // BootstrapModeFile represents a file-based bootstrap mode
        BootstrapModeFile
        // BootstrapModeHosts represents a list of hosts passed in the configuration
        BootstrapModeHosts
        // BootstrapModeCustom represents a custom bootstrap mode
        BootstrapModeCustom
        // BootstrapModeDNS represents a list of hosts passed in the configuration
        // to be resolved, and the resulting addresses are used for bootstrap
        BootstrapModeDNS
        // BootstrapModeDNSSRV represents a list of DNS hosts passed in the configuration
        // to resolve secondary addresses that DNS SRV record would return resulting in
        // a host list that will contain multiple dynamic addresses and their unique ports
        BootstrapModeDNSSRV
)

const (
        defaultMaxJoinDuration = 10 * time.Second
)

// Config contains the ringpop config items
type Config struct {
        // Name to be used in ringpop advertisement
        Name string `yaml:"name" validate:"nonzero"`
        // BootstrapMode is a enum that defines the ringpop bootstrap method, currently supports: hosts, files, custom, dns, and dns-srv
        BootstrapMode BootstrapMode `yaml:"bootstrapMode"`
        // BootstrapHosts is a list of seed hosts to be used for ringpop bootstrap
        BootstrapHosts []string `yaml:"bootstrapHosts"`
        // BootstrapFile is the file path to be used for ringpop bootstrap
        BootstrapFile string `yaml:"bootstrapFile"`
        // MaxJoinDuration is the max wait time to join the ring
        MaxJoinDuration time.Duration `yaml:"maxJoinDuration"`
        // Custom discovery provider, cannot be specified through yaml
        DiscoveryProvider discovery.DiscoverProvider `yaml:"-"`
}

func (rpConfig *Config) validate() error {
        if len(rpConfig.Name) == 0 {
                return fmt.Errorf("ringpop config missing `name` param")
        }

        if rpConfig.MaxJoinDuration == 0 {
                rpConfig.MaxJoinDuration = defaultMaxJoinDuration
        }

        return validateBootstrapMode(rpConfig)
}

// UnmarshalYAML is called by the yaml package to convert
// the config YAML into a BootstrapMode.
func (m *BootstrapMode) UnmarshalYAML(
        unmarshal func(interface{}) error,
) error {

        var s string
        if err := unmarshal(&s); err != nil {
                return err
        }
        var err error
        *m, err = parseBootstrapMode(s)
        return err
}

// parseBootstrapMode reads a string value and returns a bootstrap mode.
func parseBootstrapMode(
        mode string,
) (BootstrapMode, error) {

        switch strings.ToLower(mode) {
        case "hosts":
                return BootstrapModeHosts, nil
        case "file":
                return BootstrapModeFile, nil
        case "custom":
                return BootstrapModeCustom, nil
        case "dns":
                return BootstrapModeDNS, nil
        case "dns-srv":
                return BootstrapModeDNSSRV, nil
        }
        return BootstrapModeNone, fmt.Errorf("invalid ringpop bootstrap mode %q", mode)
}

func validateBootstrapMode(
        rpConfig *Config,
) error {

        switch rpConfig.BootstrapMode {
        case BootstrapModeFile:
                if len(rpConfig.BootstrapFile) == 0 {
                        return fmt.Errorf("ringpop config missing bootstrap file param")
                }
        case BootstrapModeHosts, BootstrapModeDNS, BootstrapModeDNSSRV:
                if len(rpConfig.BootstrapHosts) == 0 {
                        return fmt.Errorf("ringpop config missing boostrap hosts param")
                }
        case BootstrapModeCustom:
                if rpConfig.DiscoveryProvider == nil {
                        return fmt.Errorf("ringpop bootstrapMode is set to custom but discoveryProvider is nil")
                }
        default:
                return fmt.Errorf("ringpop config with unknown boostrap mode %q", rpConfig.BootstrapMode)
        }
        return nil
}

type dnsHostResolver interface {
        LookupHost(ctx context.Context, host string) (addrs []string, err error)
        LookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*net.SRV, err error)
}

type dnsProvider struct {
        UnresolvedHosts []string
        Resolver        dnsHostResolver
        Logger          log.Logger
}

func newDNSProvider(
        hosts []string,
        resolver dnsHostResolver,
        logger log.Logger,
) *dnsProvider {

        set := map[string]struct{}{}
        for _, hostport := range hosts {
                set[hostport] = struct{}{}
        }

        var keys []string
        for key := range set {
                keys = append(keys, key)
        }
        return &dnsProvider{
                UnresolvedHosts: keys,
                Resolver:        resolver,
                Logger:          logger,
        }
}

func (provider *dnsProvider) Hosts() ([]string, error) {
        var results []string
        resolvedHosts := map[string][]string{}
        for _, hostPort := range provider.UnresolvedHosts {
                host, port, err := net.SplitHostPort(hostPort)
                if err != nil {
                        provider.Logger.Warn("could not split host and port", tag.Address(hostPort), tag.Error(err))
                        continue
                }

                resolved, exists := resolvedHosts[host]
                if !exists {
                        resolved, err = provider.Resolver.LookupHost(context.Background(), host)
                        if err != nil {
                                provider.Logger.Warn("could not resolve host", tag.Address(host), tag.Error(err))
                                continue
                        }
                        resolvedHosts[host] = resolved
                }
                for _, r := range resolved {
                        results = append(results, net.JoinHostPort(r, port))
                }
        }
        if len(results) == 0 {
                return nil, errors.New("no hosts found, and bootstrap requires at least one")
        }
        return results, nil
}

type dnsSRVProvider struct {
        UnresolvedHosts []string
        Resolver        dnsHostResolver
        Logger          log.Logger
}

func newDNSSRVProvider(
        hosts []string,
        resolver dnsHostResolver,
        logger log.Logger,
) *dnsSRVProvider {

        set := map[string]struct{}{}
        for _, hostport := range hosts {
                set[hostport] = struct{}{}
        }

        var keys []string
        for key := range set {
                keys = append(keys, key)
        }

        return &dnsSRVProvider{
                UnresolvedHosts: keys,
                Resolver:        resolver,
                Logger:          logger,
        }
}

func (provider *dnsSRVProvider) Hosts() ([]string, error) {
        var results []string
        resolvedHosts := map[string][]string{}

        for _, host := range provider.UnresolvedHosts {
                hostParts := strings.Split(host, ".")
                if len(hostParts) <= 2 {
                        return nil, fmt.Errorf("could not seperate host from domain %q", host)
                }
                serviceName := hostParts[0]
                domain := strings.Join(hostParts[1:], ".")
                resolved, exists := resolvedHosts[serviceName]
                if !exists {
                        _, srvs, err := provider.Resolver.LookupSRV(context.Background(), serviceName, "tcp", domain)

                        if err != nil {
                                return nil, fmt.Errorf("could not resolve host: %s.%s", serviceName, domain)
                        }

                        var targets []string
                        for _, s := range srvs {
                                addrs, err := provider.Resolver.LookupHost(context.Background(), s.Target)

                                if err != nil {
                                        provider.Logger.Warn("could not resolve srv dns host", tag.Address(s.Target), tag.Error(err))
                                        continue
                                }
                                for _, a := range addrs {
                                        targets = append(targets, net.JoinHostPort(a, fmt.Sprintf("%d", s.Port)))
                                }
                        }
                        resolvedHosts[serviceName] = targets
                        resolved = targets
                }

                results = append(results, resolved...)
        }

        if len(results) == 0 {
                return nil, errors.New("no hosts found, and bootstrap requires at least one")
        }
        return results, nil
}

func newDiscoveryProvider(
        cfg *Config,
        logger log.Logger,
) (discovery.DiscoverProvider, error) {

        if cfg.DiscoveryProvider != nil {
                // custom discovery provider takes first precedence
                return cfg.DiscoveryProvider, nil
        }

        switch cfg.BootstrapMode {
        case BootstrapModeHosts:
                return statichosts.New(cfg.BootstrapHosts...), nil
        case BootstrapModeFile:
                return jsonfile.New(cfg.BootstrapFile), nil
        case BootstrapModeDNS:
                return newDNSProvider(cfg.BootstrapHosts, net.DefaultResolver, logger), nil
        case BootstrapModeDNSSRV:
                return newDNSSRVProvider(cfg.BootstrapHosts, net.DefaultResolver, logger), nil
        }
        return nil, fmt.Errorf("unknown bootstrap mode %q", cfg.BootstrapMode)
}

1	// Copyright (c) 2017 Uber Technologies, Inc.
2	//
3	// Permission is hereby granted, free of charge, to any person obtaining a copy
4	// of this software and associated documentation files (the "Software"), to deal
5	// in the Software without restriction, including without limitation the rights
6	// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7	// copies of the Software, and to permit persons to whom the Software is
8	// furnished to do so, subject to the following conditions:
9	//
10	// The above copyright notice and this permission notice shall be included in
11	// all copies or substantial portions of the Software.
12	//
13	// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14	// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15	// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16	// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17	// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18	// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19	// THE SOFTWARE.
20
21	package ringpopprovider
22
23	import (
24	"context"
25	"errors"
26	"fmt"
27	"net"
28	"strings"
29	"time"
30
31	"github.com/uber/ringpop-go/discovery"
32	"github.com/uber/ringpop-go/discovery/jsonfile"
33	"github.com/uber/ringpop-go/discovery/statichosts"
34
35	"github.com/uber/cadence/common/log"
36	"github.com/uber/cadence/common/log/tag"
37	)
38
39	// BootstrapMode is an enum type for ringpop bootstrap mode
40	type BootstrapMode int
41
42	const (
43	// BootstrapModeNone represents a bootstrap mode set to nothing or invalid
44	BootstrapModeNone BootstrapMode = iota
45	// BootstrapModeFile represents a file-based bootstrap mode
46	BootstrapModeFile
47	// BootstrapModeHosts represents a list of hosts passed in the configuration
48	BootstrapModeHosts
49	// BootstrapModeCustom represents a custom bootstrap mode
50	BootstrapModeCustom
51	// BootstrapModeDNS represents a list of hosts passed in the configuration
52	// to be resolved, and the resulting addresses are used for bootstrap
53	BootstrapModeDNS
54	// BootstrapModeDNSSRV represents a list of DNS hosts passed in the configuration
55	// to resolve secondary addresses that DNS SRV record would return resulting in
56	// a host list that will contain multiple dynamic addresses and their unique ports
57	BootstrapModeDNSSRV
58	)
59
60	const (
61	defaultMaxJoinDuration = 10 * time.Second
62	)
63
64	// Config contains the ringpop config items
65	type Config struct {
66	// Name to be used in ringpop advertisement
67	Name string `yaml:"name" validate:"nonzero"`
68	// BootstrapMode is a enum that defines the ringpop bootstrap method, currently supports: hosts, files, custom, dns, and dns-srv
69	BootstrapMode BootstrapMode `yaml:"bootstrapMode"`
70	// BootstrapHosts is a list of seed hosts to be used for ringpop bootstrap
71	BootstrapHosts []string `yaml:"bootstrapHosts"`
72	// BootstrapFile is the file path to be used for ringpop bootstrap
73	BootstrapFile string `yaml:"bootstrapFile"`
74	// MaxJoinDuration is the max wait time to join the ring
75	MaxJoinDuration time.Duration `yaml:"maxJoinDuration"`
76	// Custom discovery provider, cannot be specified through yaml
77	DiscoveryProvider discovery.DiscoverProvider `yaml:"-"`
78	}
79
80	func (rpConfig *Config) validate() error {	9✔
81	if len(rpConfig.Name) == 0 {	10✔
82	return fmt.Errorf("ringpop config missing `name` param")	1✔
83	}	1✔
84
85	if rpConfig.MaxJoinDuration == 0 {	9✔
86	rpConfig.MaxJoinDuration = defaultMaxJoinDuration	1✔
87	}	1✔
88
89	return validateBootstrapMode(rpConfig)	8✔
90	}
91
92	// UnmarshalYAML is called by the yaml package to convert
93	// the config YAML into a BootstrapMode.
94	func (m *BootstrapMode) UnmarshalYAML(
95	unmarshal func(interface{}) error,
96	) error {	5✔
97		5✔
98	var s string	5✔
99	if err := unmarshal(&s); err != nil {	5✔
100	return err	×
101	}	×
102	var err error	5✔
103	*m, err = parseBootstrapMode(s)	5✔
104	return err	5✔
105	}
106
107	// parseBootstrapMode reads a string value and returns a bootstrap mode.
108	func parseBootstrapMode(
109	mode string,
110	) (BootstrapMode, error) {	6✔
111		6✔
112	switch strings.ToLower(mode) {	6✔
113	case "hosts":	1✔
114	return BootstrapModeHosts, nil	1✔
115	case "file":	1✔
116	return BootstrapModeFile, nil	1✔
117	case "custom":	1✔
118	return BootstrapModeCustom, nil	1✔
119	case "dns":	1✔
120	return BootstrapModeDNS, nil	1✔
121	case "dns-srv":	1✔
122	return BootstrapModeDNSSRV, nil	1✔
123	}
124	return BootstrapModeNone, fmt.Errorf("invalid ringpop bootstrap mode %q", mode)	1✔
125	}
126
127	func validateBootstrapMode(
128	rpConfig *Config,
129	) error {	8✔
130		8✔
131	switch rpConfig.BootstrapMode {	8✔
132	case BootstrapModeFile:	1✔
133	if len(rpConfig.BootstrapFile) == 0 {	1✔
134	return fmt.Errorf("ringpop config missing bootstrap file param")	×
135	}	×
136	case BootstrapModeHosts, BootstrapModeDNS, BootstrapModeDNSSRV:	3✔
137	if len(rpConfig.BootstrapHosts) == 0 {	3✔
138	return fmt.Errorf("ringpop config missing boostrap hosts param")	×
139	}	×
140	case BootstrapModeCustom:	2✔
141	if rpConfig.DiscoveryProvider == nil {	3✔
142	return fmt.Errorf("ringpop bootstrapMode is set to custom but discoveryProvider is nil")	1✔
143	}	1✔
144	default:	2✔
145	return fmt.Errorf("ringpop config with unknown boostrap mode %q", rpConfig.BootstrapMode)	2✔
146	}
147	return nil	5✔
148	}
149
150	type dnsHostResolver interface {
151	LookupHost(ctx context.Context, host string) (addrs []string, err error)
152	LookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*net.SRV, err error)
153	}
154
155	type dnsProvider struct {
156	UnresolvedHosts []string
157	Resolver dnsHostResolver
158	Logger log.Logger
159	}
160
161	func newDNSProvider(
162	hosts []string,
163	resolver dnsHostResolver,
164	logger log.Logger,
165	) *dnsProvider {	3✔
166		3✔
167	set := map[string]struct{}{}	3✔
168	for _, hostport := range hosts {	18✔
169	set[hostport] = struct{}{}	15✔
170	}	15✔
171
172	var keys []string	3✔
173	for key := range set {	15✔
174	keys = append(keys, key)	12✔
175	}	12✔
176	return &dnsProvider{	3✔
177	UnresolvedHosts: keys,	3✔
178	Resolver: resolver,	3✔
179	Logger: logger,	3✔
180	}	3✔
181	}
182
183	func (provider *dnsProvider) Hosts() ([]string, error) {	3✔
184	var results []string	3✔
185	resolvedHosts := map[string][]string{}	3✔
186	for _, hostPort := range provider.UnresolvedHosts {	15✔
187	host, port, err := net.SplitHostPort(hostPort)	12✔
188	if err != nil {	18✔
189	provider.Logger.Warn("could not split host and port", tag.Address(hostPort), tag.Error(err))	6✔
190	continue	6✔
191	}
192
193	resolved, exists := resolvedHosts[host]	6✔
194	if !exists {	11✔
195	resolved, err = provider.Resolver.LookupHost(context.Background(), host)	5✔
196	if err != nil {	9✔
197	provider.Logger.Warn("could not resolve host", tag.Address(host), tag.Error(err))	4✔
198	continue	4✔
199	}
200	resolvedHosts[host] = resolved	1✔
201	}
202	for _, r := range resolved {	6✔
203	results = append(results, net.JoinHostPort(r, port))	4✔
204	}	4✔
205	}
206	if len(results) == 0 {	5✔
207	return nil, errors.New("no hosts found, and bootstrap requires at least one")	2✔
208	}	2✔
209	return results, nil	1✔
210	}
211
212	type dnsSRVProvider struct {
213	UnresolvedHosts []string
214	Resolver dnsHostResolver
215	Logger log.Logger
216	}
217
218	func newDNSSRVProvider(
219	hosts []string,
220	resolver dnsHostResolver,
221	logger log.Logger,
222	) *dnsSRVProvider {	1✔
223		1✔
224	set := map[string]struct{}{}	1✔
225	for _, hostport := range hosts {	6✔
226	set[hostport] = struct{}{}	5✔
227	}	5✔
228
229	var keys []string	1✔
230	for key := range set {	5✔
231	keys = append(keys, key)	4✔
232	}	4✔
233
234	return &dnsSRVProvider{	1✔
235	UnresolvedHosts: keys,	1✔
236	Resolver: resolver,	1✔
237	Logger: logger,	1✔
238	}	1✔
239	}
240
241	func (provider *dnsSRVProvider) Hosts() ([]string, error) {	3✔
242	var results []string	3✔
243	resolvedHosts := map[string][]string{}	3✔
244		3✔
245	for _, host := range provider.UnresolvedHosts {	9✔
246	hostParts := strings.Split(host, ".")	6✔
247	if len(hostParts) <= 2 {	8✔
248	return nil, fmt.Errorf("could not seperate host from domain %q", host)	2✔
249	}	2✔
250	serviceName := hostParts[0]	4✔
251	domain := strings.Join(hostParts[1:], ".")	4✔
252	resolved, exists := resolvedHosts[serviceName]	4✔
253	if !exists {	8✔
254	_, srvs, err := provider.Resolver.LookupSRV(context.Background(), serviceName, "tcp", domain)	4✔
255		4✔
256	if err != nil {	4✔
257	return nil, fmt.Errorf("could not resolve host: %s.%s", serviceName, domain)	×
258	}	×
259
260	var targets []string	4✔
261	for _, s := range srvs {	12✔
262	addrs, err := provider.Resolver.LookupHost(context.Background(), s.Target)	8✔
263		8✔
264	if err != nil {	8✔
265	provider.Logger.Warn("could not resolve srv dns host", tag.Address(s.Target), tag.Error(err))	×
266	continue	×
267	}
268	for _, a := range addrs {	20✔
269	targets = append(targets, net.JoinHostPort(a, fmt.Sprintf("%d", s.Port)))	12✔
270	}	12✔
271	}
272	resolvedHosts[serviceName] = targets	4✔
273	resolved = targets	4✔
274	}
275
276	results = append(results, resolved...)	4✔
277	}
278
279	if len(results) == 0 {	1✔
280	return nil, errors.New("no hosts found, and bootstrap requires at least one")	×
281	}	×
282	return results, nil	1✔
283	}
284
285	func newDiscoveryProvider(
286	cfg *Config,
287	logger log.Logger,
288	) (discovery.DiscoverProvider, error) {	×
289		×
290	if cfg.DiscoveryProvider != nil {	×
291	// custom discovery provider takes first precedence	×
292	return cfg.DiscoveryProvider, nil	×
293	}	×
294
295	switch cfg.BootstrapMode {	×
296	case BootstrapModeHosts:	×
297	return statichosts.New(cfg.BootstrapHosts...), nil	×
298	case BootstrapModeFile:	×
299	return jsonfile.New(cfg.BootstrapFile), nil	×
300	case BootstrapModeDNS:	×
301	return newDNSProvider(cfg.BootstrapHosts, net.DefaultResolver, logger), nil	×
302	case BootstrapModeDNSSRV:	×
303	return newDNSSRVProvider(cfg.BootstrapHosts, net.DefaultResolver, logger), nil	×
304	}
305	return nil, fmt.Errorf("unknown bootstrap mode %q", cfg.BootstrapMode)	×
306	}

uber / cadence / 01875e2f-959c-4c4d-87af-1d7805759bcc

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous