15063404311

Committed 16 May 2025 07:46AM UTC coverage: 27.73% (+1.0%) from 26.721%

Build # 15063404311

Build Type

push

github

Committed by

jbeemster

Commit Message

Prepared for release

Run Details

353 of 1273 relevant lines covered (27.73%)

2.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

31.67

/sql_runner/main.go

// Copyright (c) 2015-2025 Snowplow Analytics Ltd. All rights reserved.
//
// This program is licensed to you under the Apache License Version 2.0,
// and you may not use this file except in compliance with the Apache License Version 2.0.
// You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the Apache License Version 2.0 is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
package main

import (
        "errors"
        "fmt"
        "log"
        "os"
        "path/filepath"
        "strings"

        "github.com/kardianos/osext"
)

const (
        cliName        = "sql-runner"
        cliDescription = `Run playbooks of SQL scripts in series and parallel on Redshift, Postgres, BigQuery and Snowflake`
        cliVersion     = "0.11.0"

        sqlrootBinary        = "BINARY"
        sqlrootPlaybook      = "PLAYBOOK"
        sqlrootPlaybookChild = "PLAYBOOK_CHILD"
)

// main is the entry point for the application
func main() {

        options := processFlags()

        lockFile, lockErr := LockFileFromOptions(options)
        if lockErr != nil {
                log.Printf("Error: %s", lockErr.Error())
                os.Exit(3)
        }

        pbp, pbpErr := PlaybookProviderFromOptions(options)
        if pbpErr != nil {
                log.Fatalf("Could not determine playbook source: %s", pbpErr.Error())
        }

        pb, err := pbp.GetPlaybook()
        if err != nil {
                log.Fatalf("Error getting playbook: %s", err.Error())
        }

        if err := pb.Validate(); err != nil {
                log.Fatalf("Invalid playbook: %s", err.Error())
        }

        pb.MergeCLIVariables(options.variables)

        sp, spErr := SQLProviderFromOptions(options)
        if spErr != nil {
                log.Fatalf("Could not determine sql source: %s", spErr.Error())
        }

        // Lock it up...
        if lockFile != nil {
                lockErr2 := lockFile.Lock()
                if lockErr2 != nil {
                        log.Fatalf("Error making lock: %s", lockErr2.Error())
                }
        }

        statuses := Run(*pb, sp, options.fromStep, options.runQuery, options.dryRun, options.fillTemplates, options.showQueryOutput)
        code, message := review(statuses)

        // Unlock on success and soft-lock
        if lockFile != nil {
                if code == 0 || code == 8 || lockFile.SoftLock {
                        lockFile.Unlock()
                }
        }

        log.Print(message)
        os.Exit(code)
}

// processFlags parses the arguments provided to
// the main function.
func processFlags() Options {

        var options Options = NewOptions()
        var fs = options.GetFlagSet()
        fs.Parse(os.Args[1:])

        if options.version {
                fmt.Printf("%s version: %s\n", cliName, cliVersion)
                os.Exit(0)
        }

        if len(os.Args[1:]) == 0 || options.help {
                fmt.Printf("%s version: %s\n", cliName, cliVersion)
                fmt.Println(cliDescription)
                fmt.Println("Usage:")
                fs.PrintDefaults()
                os.Exit(0)
        }

        if options.checkLock != "" {
                lockFile, lockErr := LockFileFromOptions(options)
                if lockErr != nil {
                        log.Printf("Error: %s found, previous run failed or is ongoing", lockFile.Path)
                        os.Exit(3)
                } else {
                        log.Printf("Success: %s does not exist", lockFile.Path)
                        os.Exit(0)
                }
        }

        if options.deleteLock != "" {
                lockFile, lockErr := LockFileFromOptions(options)
                if lockErr != nil {
                        unlockErr := lockFile.Unlock()
                        if unlockErr != nil {
                                log.Printf("Error: %s found but could not delete: %s", lockFile.Path, unlockErr.Error())
                                os.Exit(1)
                        } else {
                                log.Printf("Success: %s found and deleted", lockFile.Path)
                                os.Exit(0)
                        }
                } else {
                        log.Printf("Error: %s does not exist, nothing to delete", lockFile.Path)
                        os.Exit(1)
                }
        }

        if options.playbook == "" {
                fmt.Println("required flag not defined: -playbook")
                os.Exit(2)
        }

        sr, err := resolveSQLRoot(options.sqlroot, options.playbook, options.consul, options.consulOnlyForLock)
        if err != nil {
                fmt.Printf("Error resolving -sqlroot: %s\n%s\n", options.sqlroot, err)
                os.Exit(2)
        }
        options.sqlroot = sr // Yech, mutate in place

        return options
}

// --- Options resolvers

// PlaybookProviderFromOptions returns a provider of the Playbook
// based on flags passed in
func PlaybookProviderFromOptions(options Options) (PlaybookProvider, error) {
        if options.consulOnlyForLock {
                return NewYAMLFilePlaybookProvider(options.playbook, options.variables), nil
        } else if options.consul != "" {
                return NewConsulPlaybookProvider(options.consul, options.playbook, options.variables), nil
        } else if options.playbook != "" {
                return NewYAMLFilePlaybookProvider(options.playbook, options.variables), nil
        } else {
                return nil, errors.New("Cannot determine provider for playbook")
        }
}

// SQLProviderFromOptions returns a provider of SQL files
// based on flags passed in
func SQLProviderFromOptions(options Options) (SQLProvider, error) {
        if options.consulOnlyForLock {
                return NewFileSQLProvider(options.sqlroot), nil
        } else if options.consul != "" {
                return NewConsulSQLProvider(options.consul, options.sqlroot), nil
        } else if options.playbook != "" {
                return NewFileSQLProvider(options.sqlroot), nil
        } else {
                return nil, errors.New("Cannot determine provider for sql")
        }
}

// LockFileFromOptions will check if a LockFile already
// exists and will then either:
// 1. Raise an error
// 2. Set a new lock
func LockFileFromOptions(options Options) (*LockFile, error) {

        // Do nothing if dry-run
        if options.dryRun == true {
                return nil, nil
        }

        var lockPath string
        var isSoftLock bool

        if options.lock != "" {
                lockPath = options.lock
                isSoftLock = false
        } else if options.softLock != "" {
                lockPath = options.softLock
                isSoftLock = true
        } else if options.checkLock != "" {
                lockPath = options.checkLock
                isSoftLock = false
        } else if options.deleteLock != "" {
                lockPath = options.deleteLock
                isSoftLock = false
        } else {
                // no-op
                return nil, nil
        }

        lockFile, err := InitLockFile(lockPath, isSoftLock, options.consul)

        return &lockFile, err
}

// --- SQLRoot resolvers

// resolveSQLRoot returns the path to our SQL scripts
func resolveSQLRoot(sqlroot string, playbookPath string, consulAddress string, consulOnlyForLock bool) (string, error) {
        consulErr1 := fmt.Errorf("Cannot use %s option with -consul argument", sqlroot)
        consulErr2 := fmt.Errorf("Cannot use %s option without -consul argument", sqlroot)
        consulErr3 := fmt.Errorf("Cannot use %s option with -consulOnlyForLock argument", sqlroot)

        if consulOnlyForLock {
                switch sqlroot {
                case sqlrootBinary:
                        return osext.ExecutableFolder()
                case sqlrootPlaybook:
                        return filepath.Abs(filepath.Dir(playbookPath))
                case sqlrootPlaybookChild:
                        return "", consulErr3
                default:
                        return sqlroot, nil
                }
        }

        switch sqlroot {
        case sqlrootBinary:
                if consulAddress != "" {
                        return "", consulErr1
                }
                return osext.ExecutableFolder()
        case sqlrootPlaybook:
                if consulAddress != "" {
                        return getAbsConsulPath(playbookPath), nil
                }
                return filepath.Abs(filepath.Dir(playbookPath))
        case sqlrootPlaybookChild:
                if consulAddress != "" {
                        return playbookPath, nil
                }
                return "", consulErr2
        default:
                return sqlroot, nil
        }
}

// getAbsConsulPath returns an absolute path for Consul
// one directory up
func getAbsConsulPath(path string) string {
        strSpl := strings.Split(path, "/")
        trimSpl := strSpl[:len(strSpl)-1]
        return strings.Join(trimSpl, "/")
}

1	// Copyright (c) 2015-2025 Snowplow Analytics Ltd. All rights reserved.
2	//
3	// This program is licensed to you under the Apache License Version 2.0,
4	// and you may not use this file except in compliance with the Apache License Version 2.0.
5	// You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
6	//
7	// Unless required by applicable law or agreed to in writing,
8	// software distributed under the Apache License Version 2.0 is distributed on an
9	// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10	// See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
11	package main
12
13	import (
14	"errors"
15	"fmt"
16	"log"
17	"os"
18	"path/filepath"
19	"strings"
20
21	"github.com/kardianos/osext"
22	)
23
24	const (
25	cliName = "sql-runner"
26	cliDescription = `Run playbooks of SQL scripts in series and parallel on Redshift, Postgres, BigQuery and Snowflake`
27	cliVersion = "0.11.0"
28
29	sqlrootBinary = "BINARY"
30	sqlrootPlaybook = "PLAYBOOK"
31	sqlrootPlaybookChild = "PLAYBOOK_CHILD"
32	)
33
34	// main is the entry point for the application
35	func main() {	×
36		×
37	options := processFlags()	×
38		×
39	lockFile, lockErr := LockFileFromOptions(options)	×
40	if lockErr != nil {	×
41	log.Printf("Error: %s", lockErr.Error())	×
42	os.Exit(3)	×
43	}	×
44
45	pbp, pbpErr := PlaybookProviderFromOptions(options)	×
46	if pbpErr != nil {	×
47	log.Fatalf("Could not determine playbook source: %s", pbpErr.Error())	×
48	}	×
49
50	pb, err := pbp.GetPlaybook()	×
51	if err != nil {	×
52	log.Fatalf("Error getting playbook: %s", err.Error())	×
53	}	×
54
55	if err := pb.Validate(); err != nil {	×
56	log.Fatalf("Invalid playbook: %s", err.Error())	×
57	}	×
58
59	pb.MergeCLIVariables(options.variables)	×
60		×
61	sp, spErr := SQLProviderFromOptions(options)	×
62	if spErr != nil {	×
63	log.Fatalf("Could not determine sql source: %s", spErr.Error())	×
64	}	×
65
66	// Lock it up...
67	if lockFile != nil {	×
68	lockErr2 := lockFile.Lock()	×
69	if lockErr2 != nil {	×
70	log.Fatalf("Error making lock: %s", lockErr2.Error())	×
71	}	×
72	}
73
74	statuses := Run(*pb, sp, options.fromStep, options.runQuery, options.dryRun, options.fillTemplates, options.showQueryOutput)	×
75	code, message := review(statuses)	×
76		×
77	// Unlock on success and soft-lock	×
78	if lockFile != nil {	×
79	if code == 0 \|\| code == 8 \|\| lockFile.SoftLock {	×
80	lockFile.Unlock()	×
81	}	×
82	}
83
84	log.Print(message)	×
85	os.Exit(code)	×
86	}
87
88	// processFlags parses the arguments provided to
89	// the main function.
90	func processFlags() Options {	×
91		×
92	var options Options = NewOptions()	×
93	var fs = options.GetFlagSet()	×
94	fs.Parse(os.Args[1:])	×
95		×
96	if options.version {	×
97	fmt.Printf("%s version: %s\n", cliName, cliVersion)	×
98	os.Exit(0)	×
99	}	×
100
101	if len(os.Args[1:]) == 0 \|\| options.help {	×
102	fmt.Printf("%s version: %s\n", cliName, cliVersion)	×
103	fmt.Println(cliDescription)	×
104	fmt.Println("Usage:")	×
105	fs.PrintDefaults()	×
106	os.Exit(0)	×
107	}	×
108
109	if options.checkLock != "" {	×
110	lockFile, lockErr := LockFileFromOptions(options)	×
111	if lockErr != nil {	×
112	log.Printf("Error: %s found, previous run failed or is ongoing", lockFile.Path)	×
113	os.Exit(3)	×
114	} else {	×
115	log.Printf("Success: %s does not exist", lockFile.Path)	×
116	os.Exit(0)	×
117	}	×
118	}
119
120	if options.deleteLock != "" {	×
121	lockFile, lockErr := LockFileFromOptions(options)	×
122	if lockErr != nil {	×
123	unlockErr := lockFile.Unlock()	×
124	if unlockErr != nil {	×
125	log.Printf("Error: %s found but could not delete: %s", lockFile.Path, unlockErr.Error())	×
126	os.Exit(1)	×
127	} else {	×
128	log.Printf("Success: %s found and deleted", lockFile.Path)	×
129	os.Exit(0)	×
130	}	×
131	} else {	×
132	log.Printf("Error: %s does not exist, nothing to delete", lockFile.Path)	×
133	os.Exit(1)	×
134	}	×
135	}
136
137	if options.playbook == "" {	×
138	fmt.Println("required flag not defined: -playbook")	×
139	os.Exit(2)	×
140	}	×
141
142	sr, err := resolveSQLRoot(options.sqlroot, options.playbook, options.consul, options.consulOnlyForLock)	×
143	if err != nil {	×
144	fmt.Printf("Error resolving -sqlroot: %s\n%s\n", options.sqlroot, err)	×
145	os.Exit(2)	×
146	}	×
147	options.sqlroot = sr // Yech, mutate in place	×
148		×
149	return options	×
150	}
151
152	// --- Options resolvers
153
154	// PlaybookProviderFromOptions returns a provider of the Playbook
155	// based on flags passed in
156	func PlaybookProviderFromOptions(options Options) (PlaybookProvider, error) {	×
157	if options.consulOnlyForLock {	×
158	return NewYAMLFilePlaybookProvider(options.playbook, options.variables), nil	×
159	} else if options.consul != "" {	×
160	return NewConsulPlaybookProvider(options.consul, options.playbook, options.variables), nil	×
161	} else if options.playbook != "" {	×
162	return NewYAMLFilePlaybookProvider(options.playbook, options.variables), nil	×
163	} else {	×
164	return nil, errors.New("Cannot determine provider for playbook")	×
165	}	×
166	}
167
168	// SQLProviderFromOptions returns a provider of SQL files
169	// based on flags passed in
170	func SQLProviderFromOptions(options Options) (SQLProvider, error) {	×
171	if options.consulOnlyForLock {	×
172	return NewFileSQLProvider(options.sqlroot), nil	×
173	} else if options.consul != "" {	×
174	return NewConsulSQLProvider(options.consul, options.sqlroot), nil	×
175	} else if options.playbook != "" {	×
176	return NewFileSQLProvider(options.sqlroot), nil	×
177	} else {	×
178	return nil, errors.New("Cannot determine provider for sql")	×
179	}	×
180	}
181
182	// LockFileFromOptions will check if a LockFile already
183	// exists and will then either:
184	// 1. Raise an error
185	// 2. Set a new lock
186	func LockFileFromOptions(options Options) (*LockFile, error) {	6✔
187		6✔
188	// Do nothing if dry-run	6✔
189	if options.dryRun == true {	7✔
190	return nil, nil	1✔
191	}	1✔
192
193	var lockPath string	5✔
194	var isSoftLock bool	5✔
195		5✔
196	if options.lock != "" {	6✔
197	lockPath = options.lock	1✔
198	isSoftLock = false	1✔
199	} else if options.softLock != "" {	6✔
200	lockPath = options.softLock	1✔
201	isSoftLock = true	1✔
202	} else if options.checkLock != "" {	5✔
203	lockPath = options.checkLock	1✔
204	isSoftLock = false	1✔
205	} else if options.deleteLock != "" {	4✔
206	lockPath = options.deleteLock	1✔
207	isSoftLock = false	1✔
208	} else {	2✔
209	// no-op	1✔
210	return nil, nil	1✔
211	}	1✔
212
213	lockFile, err := InitLockFile(lockPath, isSoftLock, options.consul)	4✔
214		4✔
215	return &lockFile, err	4✔
216	}
217
218	// --- SQLRoot resolvers
219
220	// resolveSQLRoot returns the path to our SQL scripts
221	func resolveSQLRoot(sqlroot string, playbookPath string, consulAddress string, consulOnlyForLock bool) (string, error) {	7✔
222	consulErr1 := fmt.Errorf("Cannot use %s option with -consul argument", sqlroot)	7✔
223	consulErr2 := fmt.Errorf("Cannot use %s option without -consul argument", sqlroot)	7✔
224	consulErr3 := fmt.Errorf("Cannot use %s option with -consulOnlyForLock argument", sqlroot)	7✔
225		7✔
226	if consulOnlyForLock {	7✔
227	switch sqlroot {	×
228	case sqlrootBinary:	×
229	return osext.ExecutableFolder()	×
230	case sqlrootPlaybook:	×
231	return filepath.Abs(filepath.Dir(playbookPath))	×
232	case sqlrootPlaybookChild:	×
233	return "", consulErr3	×
234	default:	×
235	return sqlroot, nil	×
236	}
237	}
238
239	switch sqlroot {	7✔
240	case sqlrootBinary:	2✔
241	if consulAddress != "" {	3✔
242	return "", consulErr1	1✔
243	}	1✔
244	return osext.ExecutableFolder()	1✔
245	case sqlrootPlaybook:	2✔
246	if consulAddress != "" {	3✔
247	return getAbsConsulPath(playbookPath), nil	1✔
248	}	1✔
249	return filepath.Abs(filepath.Dir(playbookPath))	1✔
250	case sqlrootPlaybookChild:	2✔
251	if consulAddress != "" {	3✔
252	return playbookPath, nil	1✔
253	}	1✔
254	return "", consulErr2	1✔
255	default:	1✔
256	return sqlroot, nil	1✔
257	}
258	}
259
260	// getAbsConsulPath returns an absolute path for Consul
261	// one directory up
262	func getAbsConsulPath(path string) string {	1✔
263	strSpl := strings.Split(path, "/")	1✔
264	trimSpl := strSpl[:len(strSpl)-1]	1✔
265	return strings.Join(trimSpl, "/")	1✔
266	}	1✔

snowplow / sql-runner / 15063404311

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous