19729155206

Committed 27 Nov 2025 07:58AM UTC coverage: 54.432% (+0.4%) from 54.037%

Build # 19729155206

Build Type

Pull #1322

github

Committed by

leonklingele

Commit Message

chore: bring back unused util.go file as removing it is a breaking change

Pull Request Pull Request #1322: chore: remove dependency on "hashicorp/go-multierror"

Run Details

1 of 68 new or added lines in 22 files covered. (1.47%)

19 existing lines in 2 files now uncovered.

4378 of 8043 relevant lines covered (54.43%)

48.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.3

/database/spanner/spanner.go

package spanner

import (
        "context"
        "errors"
        "fmt"
        "io"
        "log"
        nurl "net/url"
        "regexp"
        "strconv"
        "strings"
        "sync/atomic"

        "cloud.google.com/go/spanner"
        sdb "cloud.google.com/go/spanner/admin/database/apiv1"
        "cloud.google.com/go/spanner/spansql"

        "github.com/golang-migrate/migrate/v4"
        "github.com/golang-migrate/migrate/v4/database"

        adminpb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
        "google.golang.org/api/iterator"
)

func init() {
        db := Spanner{}
        database.Register("spanner", &db)
}

// DefaultMigrationsTable is used if no custom table is specified
const DefaultMigrationsTable = "SchemaMigrations"

// Driver errors
var (
        ErrNilConfig      = errors.New("no config")
        ErrNoDatabaseName = errors.New("no database name")
        ErrNoSchema       = errors.New("no schema")
        ErrDatabaseDirty  = errors.New("database is dirty")
        ErrLockHeld       = errors.New("unable to obtain lock")
        ErrLockNotHeld    = errors.New("unable to release already released lock")
)

// Config used for a Spanner instance
type Config struct {
        MigrationsTable string
        DatabaseName    string
        // Whether to parse the migration DDL with spansql before
        // running them towards Spanner.
        // Parsing outputs clean DDL statements such as reformatted
        // and void of comments.
        CleanStatements bool
}

// Spanner implements database.Driver for Google Cloud Spanner
type Spanner struct {
        db *DB

        config *Config

        lock atomic.Bool
}

type DB struct {
        admin *sdb.DatabaseAdminClient
        data  *spanner.Client
}

func NewDB(admin sdb.DatabaseAdminClient, data spanner.Client) *DB {
        return &DB{
                admin: &admin,
                data:  &data,
        }
}

// WithInstance implements database.Driver
func WithInstance(instance *DB, config *Config) (database.Driver, error) {
        if config == nil {
                return nil, ErrNilConfig
        }

        if len(config.DatabaseName) == 0 {
                return nil, ErrNoDatabaseName
        }

        if len(config.MigrationsTable) == 0 {
                config.MigrationsTable = DefaultMigrationsTable
        }

        sx := &Spanner{
                db:     instance,
                config: config,
        }

        if err := sx.ensureVersionTable(); err != nil {
                return nil, err
        }

        return sx, nil
}

// Open implements database.Driver
func (s *Spanner) Open(url string) (database.Driver, error) {
        purl, err := nurl.Parse(url)
        if err != nil {
                return nil, err
        }

        ctx := context.Background()

        adminClient, err := sdb.NewDatabaseAdminClient(ctx)
        if err != nil {
                return nil, err
        }
        dbname := strings.Replace(migrate.FilterCustomQuery(purl).String(), "spanner://", "", 1)
        dataClient, err := spanner.NewClient(ctx, dbname)
        if err != nil {
                log.Fatal(err)
        }

        migrationsTable := purl.Query().Get("x-migrations-table")

        cleanQuery := purl.Query().Get("x-clean-statements")
        clean := false
        if cleanQuery != "" {
                clean, err = strconv.ParseBool(cleanQuery)
                if err != nil {
                        return nil, err
                }
        }

        db := &DB{admin: adminClient, data: dataClient}
        return WithInstance(db, &Config{
                DatabaseName:    dbname,
                MigrationsTable: migrationsTable,
                CleanStatements: clean,
        })
}

// Close implements database.Driver
func (s *Spanner) Close() error {
        s.db.data.Close()
        return s.db.admin.Close()
}

// Lock implements database.Driver but doesn't do anything because Spanner only
// enqueues the UpdateDatabaseDdlRequest.
func (s *Spanner) Lock() error {
        if swapped := s.lock.CompareAndSwap(false, true); swapped {
                return nil
        }
        return ErrLockHeld
}

// Unlock implements database.Driver but no action required, see Lock.
func (s *Spanner) Unlock() error {
        if swapped := s.lock.CompareAndSwap(true, false); swapped {
                return nil
        }
        return ErrLockNotHeld
}

// Run implements database.Driver
func (s *Spanner) Run(migration io.Reader) error {
        migr, err := io.ReadAll(migration)
        if err != nil {
                return err
        }

        stmts := []string{string(migr)}
        if s.config.CleanStatements {
                stmts, err = cleanStatements(migr)
                if err != nil {
                        return err
                }
        }

        ctx := context.Background()
        op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{
                Database:   s.config.DatabaseName,
                Statements: stmts,
        })

        if err != nil {
                return &database.Error{OrigErr: err, Err: "migration failed", Query: migr}
        }

        if err := op.Wait(ctx); err != nil {
                return &database.Error{OrigErr: err, Err: "migration failed", Query: migr}
        }

        return nil
}

// SetVersion implements database.Driver
func (s *Spanner) SetVersion(version int, dirty bool) error {
        ctx := context.Background()

        _, err := s.db.data.ReadWriteTransaction(ctx,
                func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
                        m := []*spanner.Mutation{
                                spanner.Delete(s.config.MigrationsTable, spanner.AllKeys()),
                                spanner.Insert(s.config.MigrationsTable,
                                        []string{"Version", "Dirty"},
                                        []interface{}{version, dirty},
                                )}
                        return txn.BufferWrite(m)
                })
        if err != nil {
                return &database.Error{OrigErr: err}
        }

        return nil
}

// Version implements database.Driver
func (s *Spanner) Version() (version int, dirty bool, err error) {
        ctx := context.Background()

        stmt := spanner.Statement{
                SQL: `SELECT Version, Dirty FROM ` + s.config.MigrationsTable + ` LIMIT 1`,
        }
        iter := s.db.data.Single().Query(ctx, stmt)
        defer iter.Stop()

        row, err := iter.Next()
        switch err {
        case iterator.Done:
                return database.NilVersion, false, nil
        case nil:
                var v int64
                if err = row.Columns(&v, &dirty); err != nil {
                        return 0, false, &database.Error{OrigErr: err, Query: []byte(stmt.SQL)}
                }
                version = int(v)
        default:
                return 0, false, &database.Error{OrigErr: err, Query: []byte(stmt.SQL)}
        }

        return version, dirty, nil
}

var nameMatcher = regexp.MustCompile(`(CREATE TABLE\s(\S+)\s)|(CREATE.+INDEX\s(\S+)\s)`)

// Drop implements database.Driver. Retrieves the database schema first and
// creates statements to drop the indexes and tables accordingly.
// Note: The drop statements are created in reverse order to how they're
// provided in the schema. Assuming the schema describes how the database can
// be "build up", it seems logical to "unbuild" the database simply by going the
// opposite direction. More testing
func (s *Spanner) Drop() error {
        ctx := context.Background()
        res, err := s.db.admin.GetDatabaseDdl(ctx, &adminpb.GetDatabaseDdlRequest{
                Database: s.config.DatabaseName,
        })
        if err != nil {
                return &database.Error{OrigErr: err, Err: "drop failed"}
        }
        if len(res.Statements) == 0 {
                return nil
        }

        stmts := make([]string, 0)
        for i := len(res.Statements) - 1; i >= 0; i-- {
                s := res.Statements[i]
                m := nameMatcher.FindSubmatch([]byte(s))

                if len(m) == 0 {
                        continue
                } else if tbl := m[2]; len(tbl) > 0 {
                        stmts = append(stmts, fmt.Sprintf(`DROP TABLE %s`, tbl))
                } else if idx := m[4]; len(idx) > 0 {
                        stmts = append(stmts, fmt.Sprintf(`DROP INDEX %s`, idx))
                }
        }

        op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{
                Database:   s.config.DatabaseName,
                Statements: stmts,
        })
        if err != nil {
                return &database.Error{OrigErr: err, Query: []byte(strings.Join(stmts, "; "))}
        }
        if err := op.Wait(ctx); err != nil {
                return &database.Error{OrigErr: err, Query: []byte(strings.Join(stmts, "; "))}
        }

        return nil
}

// ensureVersionTable checks if versions table exists and, if not, creates it.
// Note that this function locks the database, which deviates from the usual
// convention of "caller locks" in the Spanner type.
func (s *Spanner) ensureVersionTable() (err error) {
        if err = s.Lock(); err != nil {
                return err
        }

        defer func() {
                if e := s.Unlock(); e != nil {
                        err = errors.Join(err, e)
                }
        }()

        ctx := context.Background()
        tbl := s.config.MigrationsTable
        iter := s.db.data.Single().Read(ctx, tbl, spanner.AllKeys(), []string{"Version"})
        if err := iter.Do(func(r *spanner.Row) error { return nil }); err == nil {
                return nil
        }

        stmt := fmt.Sprintf(`CREATE TABLE %s (
    Version INT64 NOT NULL,
    Dirty    BOOL NOT NULL
        ) PRIMARY KEY(Version)`, tbl)

        op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{
                Database:   s.config.DatabaseName,
                Statements: []string{stmt},
        })

        if err != nil {
                return &database.Error{OrigErr: err, Query: []byte(stmt)}
        }
        if err := op.Wait(ctx); err != nil {
                return &database.Error{OrigErr: err, Query: []byte(stmt)}
        }

        return nil
}

func cleanStatements(migration []byte) ([]string, error) {
        // The Spanner GCP backend does not yet support comments for the UpdateDatabaseDdl RPC
        // (see https://issuetracker.google.com/issues/159730604) we use
        // spansql to parse the DDL and output valid stamements without comments
        ddl, err := spansql.ParseDDL("", string(migration))
        if err != nil {
                return nil, err
        }
        stmts := make([]string, 0, len(ddl.List))
        for _, stmt := range ddl.List {
                stmts = append(stmts, stmt.SQL())
        }
        return stmts, nil
}

1	package spanner
2
3	import (
4	"context"
5	"errors"
6	"fmt"
7	"io"
8	"log"
9	nurl "net/url"
10	"regexp"
11	"strconv"
12	"strings"
13	"sync/atomic"
14
15	"cloud.google.com/go/spanner"
16	sdb "cloud.google.com/go/spanner/admin/database/apiv1"
17	"cloud.google.com/go/spanner/spansql"
18
19	"github.com/golang-migrate/migrate/v4"
20	"github.com/golang-migrate/migrate/v4/database"
21
22	adminpb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
23	"google.golang.org/api/iterator"
24	)
25
26	func init() {	2✔
27	db := Spanner{}	2✔
28	database.Register("spanner", &db)	2✔
29	}	2✔
30
31	// DefaultMigrationsTable is used if no custom table is specified
32	const DefaultMigrationsTable = "SchemaMigrations"
33
34	// Driver errors
35	var (
36	ErrNilConfig = errors.New("no config")
37	ErrNoDatabaseName = errors.New("no database name")
38	ErrNoSchema = errors.New("no schema")
39	ErrDatabaseDirty = errors.New("database is dirty")
40	ErrLockHeld = errors.New("unable to obtain lock")
41	ErrLockNotHeld = errors.New("unable to release already released lock")
42	)
43
44	// Config used for a Spanner instance
45	type Config struct {
46	MigrationsTable string
47	DatabaseName string
48	// Whether to parse the migration DDL with spansql before
49	// running them towards Spanner.
50	// Parsing outputs clean DDL statements such as reformatted
51	// and void of comments.
52	CleanStatements bool
53	}
54
55	// Spanner implements database.Driver for Google Cloud Spanner
56	type Spanner struct {
57	db *DB
58
59	config *Config
60
61	lock atomic.Bool
62	}
63
64	type DB struct {
65	admin *sdb.DatabaseAdminClient
66	data *spanner.Client
67	}
68
69	func NewDB(admin sdb.DatabaseAdminClient, data spanner.Client) *DB {	×
70	return &DB{	×
71	admin: &admin,	×
72	data: &data,	×
73	}	×
74	}	×
75
76	// WithInstance implements database.Driver
77	func WithInstance(instance DB, config Config) (database.Driver, error) {	4✔
78	if config == nil {	4✔
79	return nil, ErrNilConfig	×
80	}	×
81
82	if len(config.DatabaseName) == 0 {	4✔
83	return nil, ErrNoDatabaseName	×
84	}	×
85
86	if len(config.MigrationsTable) == 0 {	8✔
87	config.MigrationsTable = DefaultMigrationsTable	4✔
88	}	4✔
89
90	sx := &Spanner{	4✔
91	db: instance,	4✔
92	config: config,	4✔
93	}	4✔
94		4✔
95	if err := sx.ensureVersionTable(); err != nil {	4✔
96	return nil, err	×
97	}	×
98
99	return sx, nil	4✔
100	}
101
102	// Open implements database.Driver
103	func (s *Spanner) Open(url string) (database.Driver, error) {	4✔
104	purl, err := nurl.Parse(url)	4✔
105	if err != nil {	4✔
106	return nil, err	×
107	}	×
108
109	ctx := context.Background()	4✔
110		4✔
111	adminClient, err := sdb.NewDatabaseAdminClient(ctx)	4✔
112	if err != nil {	4✔
113	return nil, err	×
114	}	×
115	dbname := strings.Replace(migrate.FilterCustomQuery(purl).String(), "spanner://", "", 1)	4✔
116	dataClient, err := spanner.NewClient(ctx, dbname)	4✔
117	if err != nil {	4✔
118	log.Fatal(err)	×
119	}	×
120
121	migrationsTable := purl.Query().Get("x-migrations-table")	4✔
122		4✔
123	cleanQuery := purl.Query().Get("x-clean-statements")	4✔
124	clean := false	4✔
125	if cleanQuery != "" {	4✔
126	clean, err = strconv.ParseBool(cleanQuery)	×
127	if err != nil {	×
128	return nil, err	×
129	}	×
130	}
131
132	db := &DB{admin: adminClient, data: dataClient}	4✔
133	return WithInstance(db, &Config{	4✔
134	DatabaseName: dbname,	4✔
135	MigrationsTable: migrationsTable,	4✔
136	CleanStatements: clean,	4✔
137	})	4✔
138	}
139
140	// Close implements database.Driver
141	func (s *Spanner) Close() error {	×
142	s.db.data.Close()	×
143	return s.db.admin.Close()	×
144	}	×
145
146	// Lock implements database.Driver but doesn't do anything because Spanner only
147	// enqueues the UpdateDatabaseDdlRequest.
148	func (s *Spanner) Lock() error {	14✔
149	if swapped := s.lock.CompareAndSwap(false, true); swapped {	26✔
150	return nil	12✔
151	}	12✔
152	return ErrLockHeld	2✔
153	}
154
155	// Unlock implements database.Driver but no action required, see Lock.
156	func (s *Spanner) Unlock() error {	12✔
157	if swapped := s.lock.CompareAndSwap(true, false); swapped {	24✔
158	return nil	12✔
159	}	12✔
160	return ErrLockNotHeld	×
161	}
162
163	// Run implements database.Driver
164	func (s *Spanner) Run(migration io.Reader) error {	12✔
165	migr, err := io.ReadAll(migration)	12✔
166	if err != nil {	12✔
167	return err	×
168	}	×
169
170	stmts := []string{string(migr)}	12✔
171	if s.config.CleanStatements {	12✔
172	stmts, err = cleanStatements(migr)	×
173	if err != nil {	×
174	return err	×
175	}	×
176	}
177
178	ctx := context.Background()	12✔
179	op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{	12✔
180	Database: s.config.DatabaseName,	12✔
181	Statements: stmts,	12✔
182	})	12✔
183		12✔
184	if err != nil {	12✔
185	return &database.Error{OrigErr: err, Err: "migration failed", Query: migr}	×
186	}	×
187
188	if err := op.Wait(ctx); err != nil {	12✔
189	return &database.Error{OrigErr: err, Err: "migration failed", Query: migr}	×
190	}	×
191
192	return nil	12✔
193	}
194
195	// SetVersion implements database.Driver
196	func (s *Spanner) SetVersion(version int, dirty bool) error {	32✔
197	ctx := context.Background()	32✔
198		32✔
199	_, err := s.db.data.ReadWriteTransaction(ctx,	32✔
200	func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {	64✔
201	m := []*spanner.Mutation{	32✔
202	spanner.Delete(s.config.MigrationsTable, spanner.AllKeys()),	32✔
203	spanner.Insert(s.config.MigrationsTable,	32✔
204	[]string{"Version", "Dirty"},	32✔
205	[]interface{}{version, dirty},	32✔
206	)}	32✔
207	return txn.BufferWrite(m)	32✔
208	})	32✔
209	if err != nil {	32✔
210	return &database.Error{OrigErr: err}	×
211	}	×
212
213	return nil	32✔
214	}
215
216	// Version implements database.Driver
217	func (s *Spanner) Version() (version int, dirty bool, err error) {	16✔
218	ctx := context.Background()	16✔
219		16✔
220	stmt := spanner.Statement{	16✔
221	SQL: `SELECT Version, Dirty FROM ` + s.config.MigrationsTable + ` LIMIT 1`,	16✔
222	}	16✔
223	iter := s.db.data.Single().Query(ctx, stmt)	16✔
224	defer iter.Stop()	16✔
225		16✔
226	row, err := iter.Next()	16✔
227	switch err {	16✔
228	case iterator.Done:	4✔
229	return database.NilVersion, false, nil	4✔
230	case nil:	12✔
231	var v int64	12✔
232	if err = row.Columns(&v, &dirty); err != nil {	12✔
233	return 0, false, &database.Error{OrigErr: err, Query: []byte(stmt.SQL)}	×
234	}	×
235	version = int(v)	12✔
236	default:	×
237	return 0, false, &database.Error{OrigErr: err, Query: []byte(stmt.SQL)}	×
238	}
239
240	return version, dirty, nil	12✔
241	}
242
243	var nameMatcher = regexp.MustCompile(`(CREATE TABLE\s(\S+)\s)\|(CREATE.+INDEX\s(\S+)\s)`)
244
245	// Drop implements database.Driver. Retrieves the database schema first and
246	// creates statements to drop the indexes and tables accordingly.
247	// Note: The drop statements are created in reverse order to how they're
248	// provided in the schema. Assuming the schema describes how the database can
249	// be "build up", it seems logical to "unbuild" the database simply by going the
250	// opposite direction. More testing
251	func (s *Spanner) Drop() error {	4✔
252	ctx := context.Background()	4✔
253	res, err := s.db.admin.GetDatabaseDdl(ctx, &adminpb.GetDatabaseDdlRequest{	4✔
254	Database: s.config.DatabaseName,	4✔
255	})	4✔
256	if err != nil {	4✔
257	return &database.Error{OrigErr: err, Err: "drop failed"}	×
258	}	×
259	if len(res.Statements) == 0 {	4✔
260	return nil	×
261	}	×
262
263	stmts := make([]string, 0)	4✔
264	for i := len(res.Statements) - 1; i >= 0; i-- {	16✔
265	s := res.Statements[i]	12✔
266	m := nameMatcher.FindSubmatch([]byte(s))	12✔
267		12✔
268	if len(m) == 0 {	12✔
269	continue	×
270	} else if tbl := m[2]; len(tbl) > 0 {	24✔
271	stmts = append(stmts, fmt.Sprintf(`DROP TABLE %s`, tbl))	12✔
272	} else if idx := m[4]; len(idx) > 0 {	12✔
273	stmts = append(stmts, fmt.Sprintf(`DROP INDEX %s`, idx))	×
274	}	×
275	}
276
277	op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{	4✔
278	Database: s.config.DatabaseName,	4✔
279	Statements: stmts,	4✔
280	})	4✔
281	if err != nil {	4✔
282	return &database.Error{OrigErr: err, Query: []byte(strings.Join(stmts, "; "))}	×
283	}	×
284	if err := op.Wait(ctx); err != nil {	4✔
285	return &database.Error{OrigErr: err, Query: []byte(strings.Join(stmts, "; "))}	×
286	}	×
287
288	return nil	4✔
289	}
290
291	// ensureVersionTable checks if versions table exists and, if not, creates it.
292	// Note that this function locks the database, which deviates from the usual
293	// convention of "caller locks" in the Spanner type.
294	func (s *Spanner) ensureVersionTable() (err error) {	4✔
295	if err = s.Lock(); err != nil {	4✔
296	return err	×
297	}	×
298
299	defer func() {	8✔
300	if e := s.Unlock(); e != nil {	4✔
NEW 301	err = errors.Join(err, e)	×
302	}	×
303	}()
304
305	ctx := context.Background()	4✔
306	tbl := s.config.MigrationsTable	4✔
307	iter := s.db.data.Single().Read(ctx, tbl, spanner.AllKeys(), []string{"Version"})	4✔
308	if err := iter.Do(func(r *spanner.Row) error { return nil }); err == nil {	4✔
309	return nil	×
310	}	×
311
312	stmt := fmt.Sprintf(`CREATE TABLE %s (	4✔
313	Version INT64 NOT NULL,	4✔
314	Dirty BOOL NOT NULL	4✔
315	) PRIMARY KEY(Version)`, tbl)	4✔
316		4✔
317	op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{	4✔
318	Database: s.config.DatabaseName,	4✔
319	Statements: []string{stmt},	4✔
320	})	4✔
321		4✔
322	if err != nil {	4✔
323	return &database.Error{OrigErr: err, Query: []byte(stmt)}	×
324	}	×
325	if err := op.Wait(ctx); err != nil {	4✔
326	return &database.Error{OrigErr: err, Query: []byte(stmt)}	×
327	}	×
328
329	return nil	4✔
330	}
331
332	func cleanStatements(migration []byte) ([]string, error) {	22✔
333	// The Spanner GCP backend does not yet support comments for the UpdateDatabaseDdl RPC	22✔
334	// (see https://issuetracker.google.com/issues/159730604) we use	22✔
335	// spansql to parse the DDL and output valid stamements without comments	22✔
336	ddl, err := spansql.ParseDDL("", string(migration))	22✔
337	if err != nil {	22✔
338	return nil, err	×
339	}	×
340	stmts := make([]string, 0, len(ddl.List))	22✔
341	for _, stmt := range ddl.List {	50✔
342	stmts = append(stmts, stmt.SQL())	28✔
343	}	28✔
344	return stmts, nil	22✔
345	}

golang-migrate / migrate / 19729155206

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous