9443940596

Committed 10 Jun 2024 07:06AM UTC coverage: 60.03%. First build

Build # 9443940596

Build Type

Pull #2364

github

Committed by

sweatybridge

Commit Message

chore: use fetcher for validating sso metadata

Pull Request Pull Request #2364: Prod deploy

Run Details

245 of 367 new or added lines in 35 files covered. (66.76%)

6904 of 11501 relevant lines covered (60.03%)

633.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.39

/internal/db/diff/diff.go

package diff

import (
        "context"
        _ "embed"
        "fmt"
        "io"
        "io/fs"
        "os"
        "path/filepath"
        "regexp"
        "strconv"
        "strings"
        "time"

        "github.com/cenkalti/backoff/v4"
        "github.com/docker/docker/api/types/container"
        "github.com/docker/docker/api/types/network"
        "github.com/docker/go-connections/nat"
        "github.com/go-errors/errors"
        "github.com/jackc/pgconn"
        "github.com/jackc/pgx/v4"
        "github.com/spf13/afero"
        "github.com/supabase/cli/internal/db/reset"
        "github.com/supabase/cli/internal/db/start"
        "github.com/supabase/cli/internal/gen/keys"
        "github.com/supabase/cli/internal/migration/apply"
        "github.com/supabase/cli/internal/migration/list"
        "github.com/supabase/cli/internal/migration/repair"
        "github.com/supabase/cli/internal/utils"
        "github.com/supabase/cli/internal/utils/parser"
)

type DiffFunc func(context.Context, string, string, []string) (string, error)

func Run(ctx context.Context, schema []string, file string, config pgconn.Config, differ DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) (err error) {
        // Sanity checks.
        if err := utils.LoadConfigFS(fsys); err != nil {
                return err
        }
        if utils.IsLocalDatabase(config) {
                if container, err := createShadowIfNotExists(ctx, fsys); err != nil {
                        return err
                } else if len(container) > 0 {
                        defer utils.DockerRemove(container)
                        if err := start.WaitForHealthyService(ctx, start.HealthTimeout, container); err != nil {
                                return err
                        }
                        if err := migrateBaseDatabase(ctx, container, fsys, options...); err != nil {
                                return err
                        }
                }
        }
        // 1. Load all user defined schemas
        if len(schema) == 0 {
                schema, err = loadSchema(ctx, config, options...)
                if err != nil {
                        return err
                }
        }
        // 3. Run migra to diff schema
        out, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, options...)
        if err != nil {
                return err
        }
        branch := keys.GetGitBranch(fsys)
        fmt.Fprintln(os.Stderr, "Finished "+utils.Aqua("supabase db diff")+" on branch "+utils.Aqua(branch)+".\n")
        if err := SaveDiff(out, file, fsys); err != nil {
                return err
        }
        drops := findDropStatements(out)
        if len(drops) > 0 {
                fmt.Fprintln(os.Stderr, "Found drop statements in schema diff. Please double check if these are expected:")
                fmt.Fprintln(os.Stderr, utils.Yellow(strings.Join(drops, "\n")))
        }
        return nil
}

func createShadowIfNotExists(ctx context.Context, fsys afero.Fs) (string, error) {
        if exists, err := afero.DirExists(fsys, utils.SchemasDir); err != nil {
                return "", errors.Errorf("failed to check schemas: %w", err)
        } else if !exists {
                return "", nil
        }
        if err := utils.AssertSupabaseDbIsRunning(); !errors.Is(err, utils.ErrNotRunning) {
                return "", err
        }
        fmt.Fprintf(os.Stderr, "Creating local database from %s...\n", utils.Bold(utils.SchemasDir))
        return CreateShadowDatabase(ctx, utils.Config.Db.Port)
}

func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) {
        var declared []string
        if err := afero.Walk(fsys, utils.SchemasDir, func(path string, info fs.FileInfo, err error) error {
                if err != nil {
                        return err
                }
                if info.Mode().IsRegular() && filepath.Ext(info.Name()) == ".sql" {
                        declared = append(declared, path)
                }
                return nil
        }); err != nil {
                return nil, errors.Errorf("failed to walk dir: %w", err)
        }
        return declared, nil
}

// https://github.com/djrobstep/migra/blob/master/migra/statements.py#L6
var dropStatementPattern = regexp.MustCompile(`(?i)drop\s+`)

func findDropStatements(out string) []string {
        lines, err := parser.SplitAndTrim(strings.NewReader(out))
        if err != nil {
                return nil
        }
        var drops []string
        for _, line := range lines {
                if dropStatementPattern.MatchString(line) {
                        drops = append(drops, line)
                }
        }
        return drops
}

func loadSchema(ctx context.Context, config pgconn.Config, options ...func(*pgx.ConnConfig)) ([]string, error) {
        conn, err := utils.ConnectByConfig(ctx, config, options...)
        if err != nil {
                return nil, err
        }
        defer conn.Close(context.Background())
        // RLS policies in auth and storage schemas can be included with -s flag
        return reset.LoadUserSchemas(ctx, conn)
}

func CreateShadowDatabase(ctx context.Context, port uint16) (string, error) {
        config := start.NewContainerConfig()
        hostPort := strconv.FormatUint(uint64(port), 10)
        hostConfig := container.HostConfig{
                PortBindings: nat.PortMap{"5432/tcp": []nat.PortBinding{{HostPort: hostPort}}},
                AutoRemove:   true,
        }
        networkingConfig := network.NetworkingConfig{}
        if utils.Config.Db.MajorVersion <= 14 {
                config.Entrypoint = nil
                hostConfig.Tmpfs = map[string]string{"/docker-entrypoint-initdb.d": ""}
        }
        return utils.DockerStart(ctx, config, hostConfig, networkingConfig, "")
}

func ConnectShadowDatabase(ctx context.Context, timeout time.Duration, options ...func(*pgx.ConnConfig)) (conn *pgx.Conn, err error) {
        // Retry until connected, cancelled, or timeout
        policy := backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Second), uint64(timeout.Seconds()))
        config := pgconn.Config{Port: utils.Config.Db.ShadowPort}
        connect := func() (*pgx.Conn, error) {
                return utils.ConnectLocalPostgres(ctx, config, options...)
        }
        return backoff.RetryWithData(connect, backoff.WithContext(policy, ctx))
}

func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
        migrations, err := list.LoadLocalMigrations(fsys)
        if err != nil {
                return err
        }
        conn, err := ConnectShadowDatabase(ctx, 10*time.Second, options...)
        if err != nil {
                return err
        }
        defer conn.Close(context.Background())
        if err := start.SetupDatabase(ctx, conn, container[:12], os.Stderr, fsys); err != nil {
                return err
        }
        return apply.MigrateUp(ctx, conn, migrations, fsys)
}

func migrateBaseDatabase(ctx context.Context, container string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
        migrations, err := loadDeclaredSchemas(fsys)
        if err != nil {
                return err
        }
        conn, err := utils.ConnectLocalPostgres(ctx, pgconn.Config{}, options...)
        if err != nil {
                return err
        }
        defer conn.Close(context.Background())
        if err := start.SetupDatabase(ctx, conn, container[:12], os.Stderr, fsys); err != nil {
                return err
        }
        for _, path := range migrations {
                fmt.Fprintln(os.Stderr, "Applying schema "+utils.Bold(path)+"...")
                migration, err := repair.NewMigrationFromFile(path, fsys)
                if err != nil {
                        return err
                }
                if err := migration.ExecBatch(ctx, conn); err != nil {
                        return err
                }
        }
        return nil
}

func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ func(context.Context, string, string, []string) (string, error), options ...func(*pgx.ConnConfig)) (string, error) {
        fmt.Fprintln(w, "Creating shadow database...")
        shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
        if err != nil {
                return "", err
        }
        defer utils.DockerRemove(shadow)
        if err := start.WaitForHealthyService(ctx, start.HealthTimeout, shadow); err != nil {
                return "", err
        }
        if err := MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil {
                return "", err
        }
        fmt.Fprintln(w, "Diffing schemas:", strings.Join(schema, ","))
        source := utils.ToPostgresURL(pgconn.Config{
                Host:     utils.Config.Hostname,
                Port:     utils.Config.Db.ShadowPort,
                User:     "postgres",
                Password: utils.Config.Db.Password,
                Database: "postgres",
        })
        target := utils.ToPostgresURL(config)
        return differ(ctx, source, target, schema)
}

1	package diff
2
3	import (
4	"context"
5	_ "embed"
6	"fmt"
7	"io"
8	"io/fs"
9	"os"
10	"path/filepath"
11	"regexp"
12	"strconv"
13	"strings"
14	"time"
15
16	"github.com/cenkalti/backoff/v4"
17	"github.com/docker/docker/api/types/container"
18	"github.com/docker/docker/api/types/network"
19	"github.com/docker/go-connections/nat"
20	"github.com/go-errors/errors"
21	"github.com/jackc/pgconn"
22	"github.com/jackc/pgx/v4"
23	"github.com/spf13/afero"
24	"github.com/supabase/cli/internal/db/reset"
25	"github.com/supabase/cli/internal/db/start"
26	"github.com/supabase/cli/internal/gen/keys"
27	"github.com/supabase/cli/internal/migration/apply"
28	"github.com/supabase/cli/internal/migration/list"
29	"github.com/supabase/cli/internal/migration/repair"
30	"github.com/supabase/cli/internal/utils"
31	"github.com/supabase/cli/internal/utils/parser"
32	)
33
34	type DiffFunc func(context.Context, string, string, []string) (string, error)
35
36	func Run(ctx context.Context, schema []string, file string, config pgconn.Config, differ DiffFunc, fsys afero.Fs, options ...func(*pgx.ConnConfig)) (err error) {	4✔
37	// Sanity checks.	4✔
38	if err := utils.LoadConfigFS(fsys); err != nil {	5✔
39	return err	1✔
40	}	1✔
41	if utils.IsLocalDatabase(config) {	3✔
42	if container, err := createShadowIfNotExists(ctx, fsys); err != nil {	×
43	return err	×
44	} else if len(container) > 0 {	×
45	defer utils.DockerRemove(container)	×
NEW 46	if err := start.WaitForHealthyService(ctx, start.HealthTimeout, container); err != nil {	×
NEW 47	return err	×
48	}	×
49	if err := migrateBaseDatabase(ctx, container, fsys, options...); err != nil {	×
50	return err	×
51	}	×
52	}
53	}
54	// 1. Load all user defined schemas
55	if len(schema) == 0 {	4✔
56	schema, err = loadSchema(ctx, config, options...)	1✔
57	if err != nil {	2✔
58	return err	1✔
59	}	1✔
60	}
61	// 3. Run migra to diff schema
62	out, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, options...)	2✔
63	if err != nil {	3✔
64	return err	1✔
65	}	1✔
66	branch := keys.GetGitBranch(fsys)	1✔
67	fmt.Fprintln(os.Stderr, "Finished "+utils.Aqua("supabase db diff")+" on branch "+utils.Aqua(branch)+".\n")	1✔
68	if err := SaveDiff(out, file, fsys); err != nil {	1✔
69	return err	×
70	}	×
71	drops := findDropStatements(out)	1✔
72	if len(drops) > 0 {	1✔
73	fmt.Fprintln(os.Stderr, "Found drop statements in schema diff. Please double check if these are expected:")	×
74	fmt.Fprintln(os.Stderr, utils.Yellow(strings.Join(drops, "\n")))	×
75	}	×
76	return nil	1✔
77	}
78
79	func createShadowIfNotExists(ctx context.Context, fsys afero.Fs) (string, error) {	×
80	if exists, err := afero.DirExists(fsys, utils.SchemasDir); err != nil {	×
81	return "", errors.Errorf("failed to check schemas: %w", err)	×
82	} else if !exists {	×
83	return "", nil	×
84	}	×
85	if err := utils.AssertSupabaseDbIsRunning(); !errors.Is(err, utils.ErrNotRunning) {	×
86	return "", err	×
87	}	×
88	fmt.Fprintf(os.Stderr, "Creating local database from %s...\n", utils.Bold(utils.SchemasDir))	×
89	return CreateShadowDatabase(ctx, utils.Config.Db.Port)	×
90	}
91
92	func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) {	×
93	var declared []string	×
94	if err := afero.Walk(fsys, utils.SchemasDir, func(path string, info fs.FileInfo, err error) error {	×
95	if err != nil {	×
96	return err	×
97	}	×
98	if info.Mode().IsRegular() && filepath.Ext(info.Name()) == ".sql" {	×
99	declared = append(declared, path)	×
100	}	×
101	return nil	×
102	}); err != nil {	×
103	return nil, errors.Errorf("failed to walk dir: %w", err)	×
104	}	×
105	return declared, nil	×
106	}
107
108	// https://github.com/djrobstep/migra/blob/master/migra/statements.py#L6
109	var dropStatementPattern = regexp.MustCompile(`(?i)drop\s+`)
110
111	func findDropStatements(out string) []string {	2✔
112	lines, err := parser.SplitAndTrim(strings.NewReader(out))	2✔
113	if err != nil {	2✔
114	return nil	×
115	}	×
116	var drops []string	2✔
117	for _, line := range lines {	6✔
118	if dropStatementPattern.MatchString(line) {	6✔
119	drops = append(drops, line)	2✔
120	}	2✔
121	}
122	return drops	2✔
123	}
124
125	func loadSchema(ctx context.Context, config pgconn.Config, options ...func(*pgx.ConnConfig)) ([]string, error) {	1✔
126	conn, err := utils.ConnectByConfig(ctx, config, options...)	1✔
127	if err != nil {	1✔
128	return nil, err	×
129	}	×
130	defer conn.Close(context.Background())	1✔
131	// RLS policies in auth and storage schemas can be included with -s flag	1✔
132	return reset.LoadUserSchemas(ctx, conn)	1✔
133	}
134
135	func CreateShadowDatabase(ctx context.Context, port uint16) (string, error) {	13✔
136	config := start.NewContainerConfig()	13✔
137	hostPort := strconv.FormatUint(uint64(port), 10)	13✔
138	hostConfig := container.HostConfig{	13✔
139	PortBindings: nat.PortMap{"5432/tcp": []nat.PortBinding{{HostPort: hostPort}}},	13✔
140	AutoRemove: true,	13✔
141	}	13✔
142	networkingConfig := network.NetworkingConfig{}	13✔
143	if utils.Config.Db.MajorVersion <= 14 {	17✔
144	config.Entrypoint = nil	4✔
145	hostConfig.Tmpfs = map[string]string{"/docker-entrypoint-initdb.d": ""}	4✔
146	}	4✔
147	return utils.DockerStart(ctx, config, hostConfig, networkingConfig, "")	13✔
148	}
149
150	func ConnectShadowDatabase(ctx context.Context, timeout time.Duration, options ...func(pgx.ConnConfig)) (conn pgx.Conn, err error) {	9✔
151	// Retry until connected, cancelled, or timeout	9✔
152	policy := backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Second), uint64(timeout.Seconds()))	9✔
153	config := pgconn.Config{Port: utils.Config.Db.ShadowPort}	9✔
154	connect := func() (*pgx.Conn, error) {	18✔
155	return utils.ConnectLocalPostgres(ctx, config, options...)	9✔
156	}	9✔
157	return backoff.RetryWithData(connect, backoff.WithContext(policy, ctx))	9✔
158	}
159
160	func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {	7✔
161	migrations, err := list.LoadLocalMigrations(fsys)	7✔
162	if err != nil {	8✔
163	return err	1✔
164	}	1✔
165	conn, err := ConnectShadowDatabase(ctx, 10*time.Second, options...)	6✔
166	if err != nil {	7✔
167	return err	1✔
168	}	1✔
169	defer conn.Close(context.Background())	5✔
170	if err := start.SetupDatabase(ctx, conn, container[:12], os.Stderr, fsys); err != nil {	7✔
171	return err	2✔
172	}	2✔
173	return apply.MigrateUp(ctx, conn, migrations, fsys)	3✔
174	}
175
176	func migrateBaseDatabase(ctx context.Context, container string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {	×
177	migrations, err := loadDeclaredSchemas(fsys)	×
178	if err != nil {	×
179	return err	×
180	}	×
181	conn, err := utils.ConnectLocalPostgres(ctx, pgconn.Config{}, options...)	×
182	if err != nil {	×
183	return err	×
184	}	×
185	defer conn.Close(context.Background())	×
186	if err := start.SetupDatabase(ctx, conn, container[:12], os.Stderr, fsys); err != nil {	×
187	return err	×
188	}	×
189	for _, path := range migrations {	×
190	fmt.Fprintln(os.Stderr, "Applying schema "+utils.Bold(path)+"...")	×
191	migration, err := repair.NewMigrationFromFile(path, fsys)	×
192	if err != nil {	×
193	return err	×
194	}	×
195	if err := migration.ExecBatch(ctx, conn); err != nil {	×
196	return err	×
197	}	×
198	}
199	return nil	×
200	}
201
202	func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ func(context.Context, string, string, []string) (string, error), options ...func(*pgx.ConnConfig)) (string, error) {	7✔
203	fmt.Fprintln(w, "Creating shadow database...")	7✔
204	shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)	7✔
205	if err != nil {	10✔
206	return "", err	3✔
207	}	3✔
208	defer utils.DockerRemove(shadow)	4✔
209	if err := start.WaitForHealthyService(ctx, start.HealthTimeout, shadow); err != nil {	5✔
210	return "", err	1✔
211	}	1✔
212	if err := MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil {	4✔
213	return "", err	1✔
214	}	1✔
215	fmt.Fprintln(w, "Diffing schemas:", strings.Join(schema, ","))	2✔
216	source := utils.ToPostgresURL(pgconn.Config{	2✔
217	Host: utils.Config.Hostname,	2✔
218	Port: utils.Config.Db.ShadowPort,	2✔
219	User: "postgres",	2✔
220	Password: utils.Config.Db.Password,	2✔
221	Database: "postgres",	2✔
222	})	2✔
223	target := utils.ToPostgresURL(config)	2✔
224	return differ(ctx, source, target, schema)	2✔
225	}

supabase / cli / 9443940596

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous