5387363788

Committed 27 Jun 2023 08:09AM UTC coverage: 61.79% (-0.08%) from 61.866%

Build # 5387363788

Build Type

Pull #1254

github

Committed by

sweatybridge

Commit Message

chore: update unit tests for shadow migrate

Pull Request Pull Request #1254: fix: apply migrations after creating shadow db

Run Details

5 of 5 new or added lines in 1 file covered. (100.0%)

4751 of 7689 relevant lines covered (61.79%)

943.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.01

/internal/db/diff/migra.go

package diff

import (
        "bytes"
        "context"
        _ "embed"
        "errors"
        "fmt"
        "io"
        "os"
        "strconv"
        "strings"
        "time"

        "github.com/docker/docker/api/types/container"
        "github.com/docker/go-connections/nat"
        "github.com/jackc/pgconn"
        "github.com/jackc/pgx/v4"
        "github.com/spf13/afero"
        "github.com/supabase/cli/internal/db/reset"
        "github.com/supabase/cli/internal/db/start"
        "github.com/supabase/cli/internal/gen/keys"
        "github.com/supabase/cli/internal/migration/apply"
        "github.com/supabase/cli/internal/migration/list"
        "github.com/supabase/cli/internal/utils"
)

var (
        //go:embed templates/migra.sh
        diffSchemaScript string
)

func RunMigra(ctx context.Context, schema []string, file string, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) (err error) {
        // Sanity checks.
        if err := utils.LoadConfigFS(fsys); err != nil {
                return err
        }
        if len(config.Password) > 0 {
                fmt.Fprintln(os.Stderr, "Connecting to remote database...")
        } else {
                fmt.Fprintln(os.Stderr, "Connecting to local database...")
                if err := utils.AssertSupabaseDbIsRunning(); err != nil {
                        return err
                }
                config.Host = "localhost"
                config.Port = uint16(utils.Config.Db.Port)
                config.User = "postgres"
                config.Password = "postgres"
                config.Database = "postgres"
        }
        // 1. Load all user defined schemas
        if len(schema) == 0 {
                schema, err = loadSchema(ctx, config, options...)
                if err != nil {
                        return err
                }
        }
        // 3. Run migra to diff schema
        out, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, options...)
        if err != nil {
                return err
        }
        branch := keys.GetGitBranch(fsys)
        fmt.Fprintln(os.Stderr, "Finished "+utils.Aqua("supabase db diff")+" on branch "+utils.Aqua(branch)+".\n")
        return SaveDiff(out, file, fsys)
}

func loadSchema(ctx context.Context, config pgconn.Config, options ...func(*pgx.ConnConfig)) (schema []string, err error) {
        var conn *pgx.Conn
        if config.Host == "localhost" && config.Port == uint16(utils.Config.Db.Port) {
                conn, err = utils.ConnectLocalPostgres(ctx, config, options...)
        } else {
                conn, err = utils.ConnectRemotePostgres(ctx, config, options...)
        }
        if err != nil {
                return schema, err
        }
        defer conn.Close(context.Background())
        return LoadUserSchemas(ctx, conn)
}

func LoadUserSchemas(ctx context.Context, conn *pgx.Conn, exclude ...string) ([]string, error) {
        if len(exclude) == 0 {
                // RLS policies in auth and storage schemas can be included with -s flag
                exclude = append([]string{
                        "auth",
                        // "extensions",
                        "pgbouncer",
                        "realtime",
                        "_realtime",
                        "storage",
                        "_analytics",
                        // Exclude functions because Webhooks support is early alpha
                        "supabase_functions",
                        "supabase_migrations",
                }, utils.SystemSchemas...)
        }
        return reset.ListSchemas(ctx, conn, exclude...)
}

func CreateShadowDatabase(ctx context.Context) (string, error) {
        config := start.NewContainerConfig()
        hostPort := strconv.FormatUint(uint64(utils.Config.Db.ShadowPort), 10)
        hostConfig := container.HostConfig{
                PortBindings: nat.PortMap{"5432/tcp": []nat.PortBinding{{HostPort: hostPort}}},
                AutoRemove:   true,
        }
        if utils.Config.Db.MajorVersion <= 14 {
                config.Entrypoint = nil
                hostConfig.Tmpfs = map[string]string{"/docker-entrypoint-initdb.d": ""}
        }
        return utils.DockerStart(ctx, config, hostConfig, "")
}

func connectShadowDatabase(ctx context.Context, timeout time.Duration, options ...func(*pgx.ConnConfig)) (conn *pgx.Conn, err error) {
        now := time.Now()
        expiry := now.Add(timeout)
        ticker := time.NewTicker(time.Second)
        defer ticker.Stop()
        // Retry until connected, cancelled, or timeout
        for t := now; t.Before(expiry); t = <-ticker.C {
                conn, err = utils.ConnectLocalPostgres(ctx, pgconn.Config{Port: uint16(utils.Config.Db.ShadowPort)}, options...)
                if err == nil || errors.Is(ctx.Err(), context.Canceled) {
                        break
                }
        }
        return conn, err
}

func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
        migrations, err := list.LoadLocalMigrations(fsys)
        if err != nil {
                return err
        }
        return MigrateShadowDatabaseVersions(ctx, container, migrations, fsys, options...)
}

func MigrateShadowDatabaseVersions(ctx context.Context, container string, migrations []string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
        conn, err := connectShadowDatabase(ctx, 10*time.Second, options...)
        if err != nil {
                return err
        }
        defer conn.Close(context.Background())
        if utils.Config.Db.MajorVersion <= 14 {
                if err := initShadow14(ctx, conn, fsys); err != nil {
                        return err
                }
        } else {
                if err := initShadow15(ctx, conn, container[:12], fsys); err != nil {
                        return err
                }
        }
        return apply.MigrateUp(ctx, conn, migrations, fsys)
}

func initShadow14(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error {
        if err := apply.BatchExecDDL(ctx, conn, strings.NewReader(utils.GlobalsSql)); err != nil {
                return err
        }
        if roles, err := fsys.Open(utils.CustomRolesPath); err == nil {
                if err := apply.BatchExecDDL(ctx, conn, roles); err != nil {
                        return err
                }
        } else if !errors.Is(err, os.ErrNotExist) {
                return err
        }
        return apply.BatchExecDDL(ctx, conn, strings.NewReader(utils.InitialSchemaSql))
}

func initShadow15(ctx context.Context, conn *pgx.Conn, shadowHost string, fsys afero.Fs) error {
        // Apply service migrations
        if err := utils.DockerRunOnceWithStream(ctx, utils.StorageImage, []string{
                "ANON_KEY=" + utils.Config.Auth.AnonKey,
                "SERVICE_KEY=" + utils.Config.Auth.ServiceRoleKey,
                "PGRST_JWT_SECRET=" + utils.Config.Auth.JwtSecret,
                fmt.Sprintf("DATABASE_URL=postgresql://supabase_storage_admin:%s@%s:5432/postgres", utils.Config.Db.Password, shadowHost),
                fmt.Sprintf("FILE_SIZE_LIMIT=%v", utils.Config.Storage.FileSizeLimit),
                "STORAGE_BACKEND=file",
                "TENANT_ID=stub",
                // TODO: https://github.com/supabase/storage-api/issues/55
                "REGION=stub",
                "GLOBAL_S3_BUCKET=stub",
        }, []string{"node", "dist/scripts/migrate-call.js"}, io.Discard, os.Stderr); err != nil {
                return err
        }
        if err := utils.DockerRunOnceWithStream(ctx, utils.GotrueImage, []string{
                "GOTRUE_LOG_LEVEL=error",
                "GOTRUE_DB_DRIVER=postgres",
                fmt.Sprintf("GOTRUE_DB_DATABASE_URL=postgresql://supabase_auth_admin:%s@%s:5432/postgres", utils.Config.Db.Password, shadowHost),
                "GOTRUE_SITE_URL=" + utils.Config.Auth.SiteUrl,
                "GOTRUE_JWT_SECRET=" + utils.Config.Auth.JwtSecret,
        }, []string{"gotrue", "migrate"}, io.Discard, os.Stderr); err != nil {
                return err
        }
        // Apply user migrations
        if roles, err := fsys.Open(utils.CustomRolesPath); err == nil {
                return apply.BatchExecDDL(ctx, conn, roles)
        } else if !errors.Is(err, os.ErrNotExist) {
                return err
        }
        return nil
}

// Diffs local database schema against shadow, dumps output to stdout.
func DiffSchemaMigra(ctx context.Context, source, target string, schema []string) (string, error) {
        env := []string{"SOURCE=" + source, "TARGET=" + target}
        // Passing in script string means command line args must be set manually, ie. "$@"
        args := "set -- " + strings.Join(schema, " ") + ";"
        cmd := []string{"/bin/sh", "-c", args + diffSchemaScript}
        var out, stderr bytes.Buffer
        if err := utils.DockerRunOnceWithConfig(
                ctx,
                container.Config{
                        Image: utils.MigraImage,
                        Env:   env,
                        Cmd:   cmd,
                },
                container.HostConfig{
                        NetworkMode: container.NetworkMode("host"),
                },
                "",
                &out,
                &stderr,
        ); err != nil {
                return "", fmt.Errorf("error diffing schema: %w:\n%s", err, stderr.String())
        }
        return out.String(), nil
}

func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, options ...func(*pgx.ConnConfig)) (string, error) {
        fmt.Fprintln(w, "Creating shadow database...")
        shadow, err := CreateShadowDatabase(ctx)
        if err != nil {
                return "", err
        }
        defer utils.DockerRemove(shadow)
        if err := MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil {
                return "", err
        }
        fmt.Fprintln(w, "Diffing schemas:", strings.Join(schema, ","))
        source := fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/postgres", utils.Config.Db.ShadowPort)
        target := utils.ToPostgresURL(config)
        return DiffSchemaMigra(ctx, source, target, schema)
}

1	package diff
2
3	import (
4	"bytes"
5	"context"
6	_ "embed"
7	"errors"
8	"fmt"
9	"io"
10	"os"
11	"strconv"
12	"strings"
13	"time"
14
15	"github.com/docker/docker/api/types/container"
16	"github.com/docker/go-connections/nat"
17	"github.com/jackc/pgconn"
18	"github.com/jackc/pgx/v4"
19	"github.com/spf13/afero"
20	"github.com/supabase/cli/internal/db/reset"
21	"github.com/supabase/cli/internal/db/start"
22	"github.com/supabase/cli/internal/gen/keys"
23	"github.com/supabase/cli/internal/migration/apply"
24	"github.com/supabase/cli/internal/migration/list"
25	"github.com/supabase/cli/internal/utils"
26	)
27
28	var (
29	//go:embed templates/migra.sh
30	diffSchemaScript string
31	)
32
33	func RunMigra(ctx context.Context, schema []string, file string, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) (err error) {	5✔
34	// Sanity checks.	5✔
35	if err := utils.LoadConfigFS(fsys); err != nil {	6✔
36	return err	1✔
37	}	1✔
38	if len(config.Password) > 0 {	7✔
39	fmt.Fprintln(os.Stderr, "Connecting to remote database...")	3✔
40	} else {	4✔
41	fmt.Fprintln(os.Stderr, "Connecting to local database...")	1✔
42	if err := utils.AssertSupabaseDbIsRunning(); err != nil {	2✔
43	return err	1✔
44	}	1✔
45	config.Host = "localhost"	×
46	config.Port = uint16(utils.Config.Db.Port)	×
47	config.User = "postgres"	×
48	config.Password = "postgres"	×
49	config.Database = "postgres"	×
50	}
51	// 1. Load all user defined schemas
52	if len(schema) == 0 {	4✔
53	schema, err = loadSchema(ctx, config, options...)	1✔
54	if err != nil {	2✔
55	return err	1✔
56	}	1✔
57	}
58	// 3. Run migra to diff schema
59	out, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, options...)	2✔
60	if err != nil {	3✔
61	return err	1✔
62	}	1✔
63	branch := keys.GetGitBranch(fsys)	1✔
64	fmt.Fprintln(os.Stderr, "Finished "+utils.Aqua("supabase db diff")+" on branch "+utils.Aqua(branch)+".\n")	1✔
65	return SaveDiff(out, file, fsys)	1✔
66	}
67
68	func loadSchema(ctx context.Context, config pgconn.Config, options ...func(*pgx.ConnConfig)) (schema []string, err error) {	1✔
69	var conn *pgx.Conn	1✔
70	if config.Host == "localhost" && config.Port == uint16(utils.Config.Db.Port) {	1✔
71	conn, err = utils.ConnectLocalPostgres(ctx, config, options...)	×
72	} else {	1✔
73	conn, err = utils.ConnectRemotePostgres(ctx, config, options...)	1✔
74	}	1✔
75	if err != nil {	1✔
76	return schema, err	×
77	}	×
78	defer conn.Close(context.Background())	1✔
79	return LoadUserSchemas(ctx, conn)	1✔
80	}
81
82	func LoadUserSchemas(ctx context.Context, conn *pgx.Conn, exclude ...string) ([]string, error) {	2✔
83	if len(exclude) == 0 {	3✔
84	// RLS policies in auth and storage schemas can be included with -s flag	1✔
85	exclude = append([]string{	1✔
86	"auth",	1✔
87	// "extensions",	1✔
88	"pgbouncer",	1✔
89	"realtime",	1✔
90	"_realtime",	1✔
91	"storage",	1✔
92	"_analytics",	1✔
93	// Exclude functions because Webhooks support is early alpha	1✔
94	"supabase_functions",	1✔
95	"supabase_migrations",	1✔
96	}, utils.SystemSchemas...)	1✔
97	}	1✔
98	return reset.ListSchemas(ctx, conn, exclude...)	2✔
99	}
100
101	func CreateShadowDatabase(ctx context.Context) (string, error) {	10✔
102	config := start.NewContainerConfig()	10✔
103	hostPort := strconv.FormatUint(uint64(utils.Config.Db.ShadowPort), 10)	10✔
104	hostConfig := container.HostConfig{	10✔
105	PortBindings: nat.PortMap{"5432/tcp": []nat.PortBinding{{HostPort: hostPort}}},	10✔
106	AutoRemove: true,	10✔
107	}	10✔
108	if utils.Config.Db.MajorVersion <= 14 {	13✔
109	config.Entrypoint = nil	3✔
110	hostConfig.Tmpfs = map[string]string{"/docker-entrypoint-initdb.d": ""}	3✔
111	}	3✔
112	return utils.DockerStart(ctx, config, hostConfig, "")	10✔
113	}
114
115	func connectShadowDatabase(ctx context.Context, timeout time.Duration, options ...func(pgx.ConnConfig)) (conn pgx.Conn, err error) {	9✔
116	now := time.Now()	9✔
117	expiry := now.Add(timeout)	9✔
118	ticker := time.NewTicker(time.Second)	9✔
119	defer ticker.Stop()	9✔
120	// Retry until connected, cancelled, or timeout	9✔
121	for t := now; t.Before(expiry); t = <-ticker.C {	18✔
122	conn, err = utils.ConnectLocalPostgres(ctx, pgconn.Config{Port: uint16(utils.Config.Db.ShadowPort)}, options...)	9✔
123	if err == nil \|\| errors.Is(ctx.Err(), context.Canceled) {	18✔
124	break	9✔
125	}
126	}
127	return conn, err	9✔
128	}
129
130	func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {	6✔
131	migrations, err := list.LoadLocalMigrations(fsys)	6✔
132	if err != nil {	6✔
133	return err	×
134	}	×
135	return MigrateShadowDatabaseVersions(ctx, container, migrations, fsys, options...)	6✔
136	}
137
138	func MigrateShadowDatabaseVersions(ctx context.Context, container string, migrations []string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {	9✔
139	conn, err := connectShadowDatabase(ctx, 10*time.Second, options...)	9✔
140	if err != nil {	10✔
141	return err	1✔
142	}	1✔
143	defer conn.Close(context.Background())	8✔
144	if utils.Config.Db.MajorVersion <= 14 {	12✔
145	if err := initShadow14(ctx, conn, fsys); err != nil {	7✔
146	return err	3✔
147	}	3✔
148	} else {	4✔
149	if err := initShadow15(ctx, conn, container[:12], fsys); err != nil {	5✔
150	return err	1✔
151	}	1✔
152	}
153	return apply.MigrateUp(ctx, conn, migrations, fsys)	4✔
154	}
155
156	func initShadow14(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error {	4✔
157	if err := apply.BatchExecDDL(ctx, conn, strings.NewReader(utils.GlobalsSql)); err != nil {	6✔
158	return err	2✔
159	}	2✔
160	if roles, err := fsys.Open(utils.CustomRolesPath); err == nil {	2✔
161	if err := apply.BatchExecDDL(ctx, conn, roles); err != nil {	×
162	return err	×
163	}	×
164	} else if !errors.Is(err, os.ErrNotExist) {	2✔
165	return err	×
166	}	×
167	return apply.BatchExecDDL(ctx, conn, strings.NewReader(utils.InitialSchemaSql))	2✔
168	}
169
170	func initShadow15(ctx context.Context, conn *pgx.Conn, shadowHost string, fsys afero.Fs) error {	4✔
171	// Apply service migrations	4✔
172	if err := utils.DockerRunOnceWithStream(ctx, utils.StorageImage, []string{	4✔
173	"ANON_KEY=" + utils.Config.Auth.AnonKey,	4✔
174	"SERVICE_KEY=" + utils.Config.Auth.ServiceRoleKey,	4✔
175	"PGRST_JWT_SECRET=" + utils.Config.Auth.JwtSecret,	4✔
176	fmt.Sprintf("DATABASE_URL=postgresql://supabase_storage_admin:%s@%s:5432/postgres", utils.Config.Db.Password, shadowHost),	4✔
177	fmt.Sprintf("FILE_SIZE_LIMIT=%v", utils.Config.Storage.FileSizeLimit),	4✔
178	"STORAGE_BACKEND=file",	4✔
179	"TENANT_ID=stub",	4✔
180	// TODO: https://github.com/supabase/storage-api/issues/55	4✔
181	"REGION=stub",	4✔
182	"GLOBAL_S3_BUCKET=stub",	4✔
183	}, []string{"node", "dist/scripts/migrate-call.js"}, io.Discard, os.Stderr); err != nil {	5✔
184	return err	1✔
185	}	1✔
186	if err := utils.DockerRunOnceWithStream(ctx, utils.GotrueImage, []string{	3✔
187	"GOTRUE_LOG_LEVEL=error",	3✔
188	"GOTRUE_DB_DRIVER=postgres",	3✔
189	fmt.Sprintf("GOTRUE_DB_DATABASE_URL=postgresql://supabase_auth_admin:%s@%s:5432/postgres", utils.Config.Db.Password, shadowHost),	3✔
190	"GOTRUE_SITE_URL=" + utils.Config.Auth.SiteUrl,	3✔
191	"GOTRUE_JWT_SECRET=" + utils.Config.Auth.JwtSecret,	3✔
192	}, []string{"gotrue", "migrate"}, io.Discard, os.Stderr); err != nil {	3✔
193	return err	×
194	}	×
195	// Apply user migrations
196	if roles, err := fsys.Open(utils.CustomRolesPath); err == nil {	3✔
197	return apply.BatchExecDDL(ctx, conn, roles)	×
198	} else if !errors.Is(err, os.ErrNotExist) {	3✔
199	return err	×
200	}	×
201	return nil	3✔
202	}
203
204	// Diffs local database schema against shadow, dumps output to stdout.
205	func DiffSchemaMigra(ctx context.Context, source, target string, schema []string) (string, error) {	2✔
206	env := []string{"SOURCE=" + source, "TARGET=" + target}	2✔
207	// Passing in script string means command line args must be set manually, ie. "$@"	2✔
208	args := "set -- " + strings.Join(schema, " ") + ";"	2✔
209	cmd := []string{"/bin/sh", "-c", args + diffSchemaScript}	2✔
210	var out, stderr bytes.Buffer	2✔
211	if err := utils.DockerRunOnceWithConfig(	2✔
212	ctx,	2✔
213	container.Config{	2✔
214	Image: utils.MigraImage,	2✔
215	Env: env,	2✔
216	Cmd: cmd,	2✔
217	},	2✔
218	container.HostConfig{	2✔
219	NetworkMode: container.NetworkMode("host"),	2✔
220	},	2✔
221	"",	2✔
222	&out,	2✔
223	&stderr,	2✔
224	); err != nil {	3✔
225	return "", fmt.Errorf("error diffing schema: %w:\n%s", err, stderr.String())	1✔
226	}	1✔
227	return out.String(), nil	1✔
228	}
229
230	func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, options ...func(*pgx.ConnConfig)) (string, error) {	5✔
231	fmt.Fprintln(w, "Creating shadow database...")	5✔
232	shadow, err := CreateShadowDatabase(ctx)	5✔
233	if err != nil {	7✔
234	return "", err	2✔
235	}	2✔
236	defer utils.DockerRemove(shadow)	3✔
237	if err := MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil {	4✔
238	return "", err	1✔
239	}	1✔
240	fmt.Fprintln(w, "Diffing schemas:", strings.Join(schema, ","))	2✔
241	source := fmt.Sprintf("postgresql://postgres:postgres@localhost:%d/postgres", utils.Config.Db.ShadowPort)	2✔
242	target := utils.ToPostgresURL(config)	2✔
243	return DiffSchemaMigra(ctx, source, target, schema)	2✔
244	}

supabase / cli / 5387363788

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous