018e86dc-121c-4962-9b30-23df2fb36173

Committed 28 Mar 2024 08:59PM UTC coverage: 65.259% (+0.007%) from 65.252%

Build # 018e86dc-121c-4962-9b30-23df2fb36173

Build Type

push

buildkite

Committed by

web-flow

Commit Message

Do not panic when setting env values (#5811)

Run Details

89 of 122 new or added lines in 11 files covered. (72.95%)

32 existing lines in 10 files now uncovered.

95429 of 146231 relevant lines covered (65.26%)

2375.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

59.18

/common/persistence/sql/sqlplugin/mysql/plugin.go

// Copyright (c) 2019 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package mysql

import (
        "bytes"
        "crypto/tls"
        "crypto/x509"
        "fmt"
        "io/ioutil"
        "net"
        "net/url"
        "strings"

        "github.com/go-sql-driver/mysql"
        "github.com/iancoleman/strcase"
        "github.com/jmoiron/sqlx"

        "github.com/uber/cadence/common/config"
        pt "github.com/uber/cadence/common/persistence/persistence-tests"
        "github.com/uber/cadence/common/persistence/sql"
        "github.com/uber/cadence/common/persistence/sql/sqldriver"
        "github.com/uber/cadence/common/persistence/sql/sqlplugin"
        "github.com/uber/cadence/environment"
)

const (
        // PluginName is the name of the plugin
        PluginName                   = "mysql"
        dsnFmt                       = "%s:%s@%v(%v)/%s"
        isolationLevelAttrName       = "transaction_isolation"
        isolationLevelAttrNameLegacy = "tx_isolation"
        defaultIsolationLevel        = "'READ-COMMITTED'"
        // customTLSName is the name used if a custom tls configuration is created
        customTLSName = "tls-custom"
)

var dsnAttrOverrides = map[string]string{
        "parseTime":       "true",
        "clientFoundRows": "true",
        "multiStatements": "true",
}

type plugin struct{}

var _ sqlplugin.Plugin = (*plugin)(nil)

func init() {
        sql.RegisterPlugin(PluginName, &plugin{})
}

// CreateDB initialize the db object
func (p *plugin) CreateDB(cfg *config.SQL) (sqlplugin.DB, error) {
        conns, err := sqldriver.CreateDBConnections(cfg, func(cfg *config.SQL) (*sqlx.DB, error) {
                return p.createSingleDBConn(cfg)
        })
        if err != nil {
                return nil, err
        }
        return newDB(conns, nil, sqlplugin.DbShardUndefined, cfg.NumShards)
}

// CreateAdminDB initialize the adminDb object
func (p *plugin) CreateAdminDB(cfg *config.SQL) (sqlplugin.AdminDB, error) {
        conns, err := sqldriver.CreateDBConnections(cfg, func(cfg *config.SQL) (*sqlx.DB, error) {
                return p.createSingleDBConn(cfg)
        })
        if err != nil {
                return nil, err
        }
        return newDB(conns, nil, sqlplugin.DbShardUndefined, cfg.NumShards)
}

func (p *plugin) createSingleDBConn(cfg *config.SQL) (*sqlx.DB, error) {
        err := registerTLSConfig(cfg)
        if err != nil {
                return nil, err
        }

        db, err := sqlx.Connect(PluginName, buildDSN(cfg))
        if err != nil {
                return nil, err
        }
        if cfg.MaxConns > 0 {
                db.SetMaxOpenConns(cfg.MaxConns)
        }
        if cfg.MaxIdleConns > 0 {
                db.SetMaxIdleConns(cfg.MaxIdleConns)
        }
        if cfg.MaxConnLifetime > 0 {
                db.SetConnMaxLifetime(cfg.MaxConnLifetime)
        }

        // Maps struct names in CamelCase to snake without need for db struct tags.
        db.MapperFunc(strcase.ToSnake)
        return db, nil
}

func registerTLSConfig(cfg *config.SQL) error {
        if cfg.TLS == nil || !cfg.TLS.Enabled {
                return nil
        }

        host, _, err := net.SplitHostPort(cfg.ConnectAddr)
        if err != nil {
                return fmt.Errorf("error in host port from ConnectAddr: %v", err)
        }

        // TODO: create a way to set MinVersion and CipherSuites via cfg.
        tlsConfig := &tls.Config{
                ServerName:         host,
                InsecureSkipVerify: !cfg.TLS.EnableHostVerification,
        }

        if cfg.TLS.CaFile != "" {
                rootCertPool := x509.NewCertPool()
                pem, err := ioutil.ReadFile(cfg.TLS.CaFile)
                if err != nil {
                        return fmt.Errorf("failed to load CA files: %v", err)
                }
                if ok := rootCertPool.AppendCertsFromPEM(pem); !ok {
                        return fmt.Errorf("failed to append CA file")
                }
                tlsConfig.RootCAs = rootCertPool
        }

        if cfg.TLS.CertFile != "" && cfg.TLS.KeyFile != "" {
                clientCert := make([]tls.Certificate, 0, 1)
                certs, err := tls.LoadX509KeyPair(
                        cfg.TLS.CertFile,
                        cfg.TLS.KeyFile,
                )
                if err != nil {
                        return fmt.Errorf("failed to load tls x509 key pair: %v", err)
                }
                clientCert = append(clientCert, certs)
                tlsConfig.Certificates = clientCert
        }

        // In order to use the TLS configuration you need to register it. Once registered you use it by specifying
        // `tls` in the connect attributes.
        err = mysql.RegisterTLSConfig(customTLSName, tlsConfig)
        if err != nil {
                return fmt.Errorf("failed to register tls config: %v", err)
        }

        if cfg.ConnectAttributes == nil {
                cfg.ConnectAttributes = map[string]string{}
        }

        // If no `tls` connect attribute is provided then we override it to our newly registered tls config automatically.
        // This allows users to simply provide a tls config without needing to remember to also set the connect attribute
        if cfg.ConnectAttributes["tls"] == "" {
                if cfg.TLS.SSLMode != "" {
                        cfg.ConnectAttributes["tls"] = cfg.TLS.SSLMode
                } else {
                        cfg.ConnectAttributes["tls"] = customTLSName
                }
        }

        return nil
}

func buildDSN(cfg *config.SQL) string {
        attrs := buildDSNAttrs(cfg)
        dsn := fmt.Sprintf(dsnFmt, cfg.User, cfg.Password, cfg.ConnectProtocol, cfg.ConnectAddr, cfg.DatabaseName)
        if attrs != "" {
                dsn = dsn + "?" + attrs
        }
        return dsn
}

func buildDSNAttrs(cfg *config.SQL) string {
        attrs := make(map[string]string, len(dsnAttrOverrides)+len(cfg.ConnectAttributes)+1)
        for k, v := range cfg.ConnectAttributes {
                k1, v1 := sanitizeAttr(k, v)
                attrs[k1] = v1
        }

        // only override isolation level if not specified
        if !hasAttr(attrs, isolationLevelAttrName) &&
                !hasAttr(attrs, isolationLevelAttrNameLegacy) {
                attrs[isolationLevelAttrName] = defaultIsolationLevel
        }

        // these attrs are always overriden
        for k, v := range dsnAttrOverrides {
                attrs[k] = v
        }

        first := true
        var buf bytes.Buffer
        for k, v := range attrs {
                if !first {
                        buf.WriteString("&")
                }
                first = false
                buf.WriteString(k)
                buf.WriteString("=")
                buf.WriteString(v)
        }
        return url.PathEscape(buf.String())
}

func hasAttr(attrs map[string]string, key string) bool {
        _, ok := attrs[key]
        return ok
}

func sanitizeAttr(inkey string, invalue string) (string, string) {
        key := strings.ToLower(strings.TrimSpace(inkey))
        value := strings.ToLower(strings.TrimSpace(invalue))
        switch key {
        case isolationLevelAttrName, isolationLevelAttrNameLegacy:
                if value[0] != '\'' { // mysql sys variable values must be enclosed in single quotes
                        value = "'" + value + "'"
                }
                return key, value
        default:
                return inkey, invalue
        }
}

const (
        testSchemaDir = "schema/mysql/v8"
)

// GetTestClusterOption return test options
func GetTestClusterOption() (*pt.TestBaseOptions, error) {
        port, err := environment.GetMySQLPort()
        if err != nil {
                return nil, err
        }
        return &pt.TestBaseOptions{
                DBPluginName: PluginName,
                DBUsername:   environment.GetMySQLUser(),
                DBPassword:   environment.GetMySQLPassword(),
                DBHost:       environment.GetMySQLAddress(),
                DBPort:       port,
                SchemaDir:    testSchemaDir,
                StoreType:    config.StoreTypeSQL,
        }, nil
}

1	// Copyright (c) 2019 Uber Technologies, Inc.
2	//
3	// Permission is hereby granted, free of charge, to any person obtaining a copy
4	// of this software and associated documentation files (the "Software"), to deal
5	// in the Software without restriction, including without limitation the rights
6	// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7	// copies of the Software, and to permit persons to whom the Software is
8	// furnished to do so, subject to the following conditions:
9	//
10	// The above copyright notice and this permission notice shall be included in
11	// all copies or substantial portions of the Software.
12	//
13	// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14	// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15	// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16	// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17	// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18	// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19	// THE SOFTWARE.
20
21	package mysql
22
23	import (
24	"bytes"
25	"crypto/tls"
26	"crypto/x509"
27	"fmt"
28	"io/ioutil"
29	"net"
30	"net/url"
31	"strings"
32
33	"github.com/go-sql-driver/mysql"
34	"github.com/iancoleman/strcase"
35	"github.com/jmoiron/sqlx"
36
37	"github.com/uber/cadence/common/config"
38	pt "github.com/uber/cadence/common/persistence/persistence-tests"
39	"github.com/uber/cadence/common/persistence/sql"
40	"github.com/uber/cadence/common/persistence/sql/sqldriver"
41	"github.com/uber/cadence/common/persistence/sql/sqlplugin"
42	"github.com/uber/cadence/environment"
43	)
44
45	const (
46	// PluginName is the name of the plugin
47	PluginName = "mysql"
48	dsnFmt = "%s:%s@%v(%v)/%s"
49	isolationLevelAttrName = "transaction_isolation"
50	isolationLevelAttrNameLegacy = "tx_isolation"
51	defaultIsolationLevel = "'READ-COMMITTED'"
52	// customTLSName is the name used if a custom tls configuration is created
53	customTLSName = "tls-custom"
54	)
55
56	var dsnAttrOverrides = map[string]string{
57	"parseTime": "true",
58	"clientFoundRows": "true",
59	"multiStatements": "true",
60	}
61
62	type plugin struct{}
63
64	var _ sqlplugin.Plugin = (*plugin)(nil)
65
66	func init() {	7✔
67	sql.RegisterPlugin(PluginName, &plugin{})	7✔
68	}	7✔
69
70	// CreateDB initialize the db object
71	func (p plugin) CreateDB(cfg config.SQL) (sqlplugin.DB, error) {	47✔
72	conns, err := sqldriver.CreateDBConnections(cfg, func(cfg config.SQL) (sqlx.DB, error) {	94✔
73	return p.createSingleDBConn(cfg)	47✔
74	})	47✔
75	if err != nil {	47✔
76	return nil, err	×
77	}	×
78	return newDB(conns, nil, sqlplugin.DbShardUndefined, cfg.NumShards)	47✔
79	}
80
81	// CreateAdminDB initialize the adminDb object
82	func (p plugin) CreateAdminDB(cfg config.SQL) (sqlplugin.AdminDB, error) {	53✔
83	conns, err := sqldriver.CreateDBConnections(cfg, func(cfg config.SQL) (sqlx.DB, error) {	106✔
84	return p.createSingleDBConn(cfg)	53✔
85	})	53✔
86	if err != nil {	53✔
87	return nil, err	×
88	}	×
89	return newDB(conns, nil, sqlplugin.DbShardUndefined, cfg.NumShards)	53✔
90	}
91
92	func (p plugin) createSingleDBConn(cfg config.SQL) (*sqlx.DB, error) {	99✔
93	err := registerTLSConfig(cfg)	99✔
94	if err != nil {	99✔
95	return nil, err	×
96	}	×
97
98	db, err := sqlx.Connect(PluginName, buildDSN(cfg))	99✔
99	if err != nil {	99✔
100	return nil, err	×
101	}	×
102	if cfg.MaxConns > 0 {	99✔
103	db.SetMaxOpenConns(cfg.MaxConns)	×
104	}	×
105	if cfg.MaxIdleConns > 0 {	99✔
106	db.SetMaxIdleConns(cfg.MaxIdleConns)	×
107	}	×
108	if cfg.MaxConnLifetime > 0 {	99✔
109	db.SetConnMaxLifetime(cfg.MaxConnLifetime)	×
110	}	×
111
112	// Maps struct names in CamelCase to snake without need for db struct tags.
113	db.MapperFunc(strcase.ToSnake)	99✔
114	return db, nil	99✔
115	}
116
117	func registerTLSConfig(cfg *config.SQL) error {	99✔
118	if cfg.TLS == nil \|\| !cfg.TLS.Enabled {	198✔
119	return nil	99✔
120	}	99✔
121
122	host, _, err := net.SplitHostPort(cfg.ConnectAddr)	×
123	if err != nil {	×
124	return fmt.Errorf("error in host port from ConnectAddr: %v", err)	×
125	}	×
126
127	// TODO: create a way to set MinVersion and CipherSuites via cfg.
128	tlsConfig := &tls.Config{	×
129	ServerName: host,	×
130	InsecureSkipVerify: !cfg.TLS.EnableHostVerification,	×
131	}	×
132		×
133	if cfg.TLS.CaFile != "" {	×
134	rootCertPool := x509.NewCertPool()	×
135	pem, err := ioutil.ReadFile(cfg.TLS.CaFile)	×
136	if err != nil {	×
137	return fmt.Errorf("failed to load CA files: %v", err)	×
138	}	×
139	if ok := rootCertPool.AppendCertsFromPEM(pem); !ok {	×
140	return fmt.Errorf("failed to append CA file")	×
141	}	×
142	tlsConfig.RootCAs = rootCertPool	×
143	}
144
145	if cfg.TLS.CertFile != "" && cfg.TLS.KeyFile != "" {	×
146	clientCert := make([]tls.Certificate, 0, 1)	×
147	certs, err := tls.LoadX509KeyPair(	×
148	cfg.TLS.CertFile,	×
149	cfg.TLS.KeyFile,	×
150	)	×
151	if err != nil {	×
152	return fmt.Errorf("failed to load tls x509 key pair: %v", err)	×
153	}	×
154	clientCert = append(clientCert, certs)	×
155	tlsConfig.Certificates = clientCert	×
156	}
157
158	// In order to use the TLS configuration you need to register it. Once registered you use it by specifying
159	// `tls` in the connect attributes.
160	err = mysql.RegisterTLSConfig(customTLSName, tlsConfig)	×
161	if err != nil {	×
162	return fmt.Errorf("failed to register tls config: %v", err)	×
163	}	×
164
165	if cfg.ConnectAttributes == nil {	×
166	cfg.ConnectAttributes = map[string]string{}	×
167	}	×
168
169	// If no `tls` connect attribute is provided then we override it to our newly registered tls config automatically.
170	// This allows users to simply provide a tls config without needing to remember to also set the connect attribute
171	if cfg.ConnectAttributes["tls"] == "" {	×
172	if cfg.TLS.SSLMode != "" {	×
173	cfg.ConnectAttributes["tls"] = cfg.TLS.SSLMode	×
174	} else {	×
175	cfg.ConnectAttributes["tls"] = customTLSName	×
176	}	×
177	}
178
179	return nil	×
180	}
181
182	func buildDSN(cfg *config.SQL) string {	104✔
183	attrs := buildDSNAttrs(cfg)	104✔
184	dsn := fmt.Sprintf(dsnFmt, cfg.User, cfg.Password, cfg.ConnectProtocol, cfg.ConnectAddr, cfg.DatabaseName)	104✔
185	if attrs != "" {	208✔
186	dsn = dsn + "?" + attrs	104✔
187	}	104✔
188	return dsn	104✔
189	}
190
191	func buildDSNAttrs(cfg *config.SQL) string {	104✔
192	attrs := make(map[string]string, len(dsnAttrOverrides)+len(cfg.ConnectAttributes)+1)	104✔
193	for k, v := range cfg.ConnectAttributes {	115✔
194	k1, v1 := sanitizeAttr(k, v)	11✔
195	attrs[k1] = v1	11✔
196	}	11✔
197
198	// only override isolation level if not specified
199	if !hasAttr(attrs, isolationLevelAttrName) &&	104✔
200	!hasAttr(attrs, isolationLevelAttrNameLegacy) {	205✔
201	attrs[isolationLevelAttrName] = defaultIsolationLevel	101✔
202	}	101✔
203
204	// these attrs are always overriden
205	for k, v := range dsnAttrOverrides {	414✔
206	attrs[k] = v	310✔
207	}	310✔
208
209	first := true	104✔
210	var buf bytes.Buffer	104✔
211	for k, v := range attrs {	525✔
212	if !first {	739✔
213	buf.WriteString("&")	318✔
214	}	318✔
215	first = false	421✔
216	buf.WriteString(k)	421✔
217	buf.WriteString("=")	421✔
218	buf.WriteString(v)	421✔
219	}
220	return url.PathEscape(buf.String())	104✔
221	}
222
223	func hasAttr(attrs map[string]string, key string) bool {	206✔
224	_, ok := attrs[key]	206✔
225	return ok	206✔
226	}	206✔
227
228	func sanitizeAttr(inkey string, invalue string) (string, string) {	11✔
229	key := strings.ToLower(strings.TrimSpace(inkey))	11✔
230	value := strings.ToLower(strings.TrimSpace(invalue))	11✔
231	switch key {	11✔
232	case isolationLevelAttrName, isolationLevelAttrNameLegacy:	3✔
233	if value[0] != '\'' { // mysql sys variable values must be enclosed in single quotes	5✔
234	value = "'" + value + "'"	2✔
235	}	2✔
236	return key, value	3✔
237	default:	8✔
238	return inkey, invalue	8✔
239	}
240	}
241
242	const (
243	testSchemaDir = "schema/mysql/v8"
244	)
245
246	// GetTestClusterOption return test options
247	func GetTestClusterOption() (*pt.TestBaseOptions, error) {	14✔
248	port, err := environment.GetMySQLPort()	14✔
249	if err != nil {	14✔
NEW 250	return nil, err	×
NEW 251	}	×
252	return &pt.TestBaseOptions{	14✔
253	DBPluginName: PluginName,	14✔
254	DBUsername: environment.GetMySQLUser(),	14✔
255	DBPassword: environment.GetMySQLPassword(),	14✔
256	DBHost: environment.GetMySQLAddress(),	14✔
257	DBPort: port,	14✔
258	SchemaDir: testSchemaDir,	14✔
259	StoreType: config.StoreTypeSQL,	14✔
260	}, nil	14✔
261	}

uber / cadence / 018e86dc-121c-4962-9b30-23df2fb36173

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous