16535713976

Committed 26 Jul 2025 03:43AM UTC coverage: 55.377%. First build

Build # 16535713976

Build Type

Pull #3920

github

Committed by

web-flow

Commit Message

Merge 0c9e4b12f into b720b1240

Pull Request Pull Request #3920: Feat/enhance drop warning

Run Details

27 of 59 new or added lines in 3 files covered. (45.76%)

6220 of 11232 relevant lines covered (55.38%)

6.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

65.75

/internal/db/diff/diff.go

package diff

import (
        "context"
        _ "embed"
        "fmt"
        "io"
        "io/fs"
        "os"
        "path/filepath"
        "regexp"
        "strconv"
        "strings"
        "time"

        "github.com/cenkalti/backoff/v4"
        "github.com/docker/docker/api/types/container"
        "github.com/docker/docker/api/types/network"
        "github.com/docker/go-connections/nat"
        "github.com/go-errors/errors"
        "github.com/jackc/pgconn"
        "github.com/jackc/pgx/v4"
        "github.com/spf13/afero"
        "github.com/supabase/cli/internal/db/start"
        "github.com/supabase/cli/internal/gen/keys"
        "github.com/supabase/cli/internal/utils"
        "github.com/supabase/cli/pkg/migration"
        "github.com/supabase/cli/pkg/parser"
)

type DiffFunc func(context.Context, string, string, []string) (string, error)

func Run(ctx context.Context, schema []string, file string, config pgconn.Config, differ DiffFunc, fsys afero.Fs, confirmDrops bool, options ...func(*pgx.ConnConfig)) (err error) {
        out, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, options...)
        if err != nil {
                return err
        }
        branch := keys.GetGitBranch(fsys)
        fmt.Fprintln(os.Stderr, "Finished "+utils.Aqua("supabase db diff")+" on branch "+utils.Aqua(branch)+".\n")

        drops := findDropStatements(out)
        if len(drops) > 0 {
                if confirmDrops {
                        if err := showDropWarningAndConfirm(ctx, drops); err != nil {
                                return err
                        }
                } else {
                        fmt.Fprintln(os.Stderr, "Found drop statements in schema diff. Please double check if these are expected:")
                        fmt.Fprintln(os.Stderr, utils.Yellow(strings.Join(drops, "\n")))
                }
        }

        if err := SaveDiff(out, file, fsys); err != nil {
                return err
        }
        return nil
}

func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) {
        if schemas := utils.Config.Db.Migrations.SchemaPaths; len(schemas) > 0 {
                return schemas.Files(afero.NewIOFS(fsys))
        }
        if exists, err := afero.DirExists(fsys, utils.SchemasDir); err != nil {
                return nil, errors.Errorf("failed to check schemas: %w", err)
        } else if !exists {
                return nil, nil
        }
        var declared []string
        if err := afero.Walk(fsys, utils.SchemasDir, func(path string, info fs.FileInfo, err error) error {
                if err != nil {
                        return err
                }
                if info.Mode().IsRegular() && filepath.Ext(info.Name()) == ".sql" {
                        declared = append(declared, path)
                }
                return nil
        }); err != nil {
                return nil, errors.Errorf("failed to walk dir: %w", err)
        }
        return declared, nil
}

// https://github.com/djrobstep/migra/blob/master/migra/statements.py#L6
var dropStatementPattern = regexp.MustCompile(`(?i)drop\s+`)

func findDropStatements(out string) []string {
        lines, err := parser.SplitAndTrim(strings.NewReader(out))
        if err != nil {
                return nil
        }
        var drops []string
        for _, line := range lines {
                if dropStatementPattern.MatchString(line) {
                        drops = append(drops, line)
                }
        }
        return drops
}

func showDropWarningAndConfirm(ctx context.Context, drops []string) error {
        fmt.Fprintln(os.Stderr, utils.Red("⚠️  DANGEROUS OPERATION DETECTED"))
        fmt.Fprintln(os.Stderr, utils.Red("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"))
        fmt.Fprintln(os.Stderr, "")
        fmt.Fprintln(os.Stderr, utils.Bold("The following DROP statements were found in your schema diff:"))
        fmt.Fprintln(os.Stderr, "")
        for _, drop := range drops {
                fmt.Fprintln(os.Stderr, "  "+utils.Red("▶ "+drop))
        }
        fmt.Fprintln(os.Stderr, "")
        fmt.Fprintln(os.Stderr, utils.Yellow("❗ These operations may cause DATA LOSS:"))
        fmt.Fprintln(os.Stderr, "  • Column renames are detected as DROP + ADD, which will lose existing data")
        fmt.Fprintln(os.Stderr, "  • Table or schema deletions will permanently remove all data")
        fmt.Fprintln(os.Stderr, "  • Consider using RENAME operations instead of DROP + ADD for columns")
        fmt.Fprintln(os.Stderr, "")
        fmt.Fprintln(os.Stderr, utils.Bold("Please review the generated migration file carefully before proceeding."))
        fmt.Fprintln(os.Stderr, "")

        console := utils.NewConsole()
        confirmed, err := console.PromptYesNo(ctx, "Do you want to continue with this potentially destructive operation?", false)
        if err != nil {
                return errors.Errorf("failed to get user confirmation: %w", err)
        }
        if !confirmed {
                return errors.New("operation cancelled by user")
        }

        fmt.Fprintln(os.Stderr, "")
        fmt.Fprintln(os.Stderr, utils.Yellow("⚠️  Proceeding with potentially destructive operation as requested."))
        fmt.Fprintln(os.Stderr, "")
        return nil
}

func loadSchema(ctx context.Context, config pgconn.Config, options ...func(*pgx.ConnConfig)) ([]string, error) {
        conn, err := utils.ConnectByConfig(ctx, config, options...)
        if err != nil {
                return nil, err
        }
        defer conn.Close(context.Background())
        // RLS policies in auth and storage schemas can be included with -s flag
        return migration.ListUserSchemas(ctx, conn)
}

func CreateShadowDatabase(ctx context.Context, port uint16) (string, error) {
        // Disable background workers in shadow database
        config := start.NewContainerConfig("-c", "max_worker_processes=0")
        hostPort := strconv.FormatUint(uint64(port), 10)
        hostConfig := container.HostConfig{
                PortBindings: nat.PortMap{"5432/tcp": []nat.PortBinding{{HostPort: hostPort}}},
                AutoRemove:   true,
        }
        networkingConfig := network.NetworkingConfig{}
        if utils.Config.Db.MajorVersion <= 14 {
                hostConfig.Tmpfs = map[string]string{"/docker-entrypoint-initdb.d": ""}
        }
        return utils.DockerStart(ctx, config, hostConfig, networkingConfig, "")
}

func ConnectShadowDatabase(ctx context.Context, timeout time.Duration, options ...func(*pgx.ConnConfig)) (conn *pgx.Conn, err error) {
        // Retry until connected, cancelled, or timeout
        policy := start.NewBackoffPolicy(ctx, timeout)
        config := pgconn.Config{Port: utils.Config.Db.ShadowPort}
        connect := func() (*pgx.Conn, error) {
                return utils.ConnectLocalPostgres(ctx, config, options...)
        }
        return backoff.RetryWithData(connect, policy)
}

// Required to bypass pg_cron check: https://github.com/citusdata/pg_cron/blob/main/pg_cron.sql#L3
const CREATE_TEMPLATE = "CREATE DATABASE contrib_regression TEMPLATE postgres"

func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
        migrations, err := migration.ListLocalMigrations(utils.MigrationsDir, afero.NewIOFS(fsys))
        if err != nil {
                return err
        }
        conn, err := ConnectShadowDatabase(ctx, 10*time.Second, options...)
        if err != nil {
                return err
        }
        defer conn.Close(context.Background())
        if err := start.SetupDatabase(ctx, conn, container[:12], os.Stderr, fsys); err != nil {
                return err
        }
        if _, err := conn.Exec(ctx, CREATE_TEMPLATE); err != nil {
                return errors.Errorf("failed to create template database: %w", err)
        }
        return migration.ApplyMigrations(ctx, migrations, conn, afero.NewIOFS(fsys))
}

func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ func(context.Context, string, string, []string) (string, error), options ...func(*pgx.ConnConfig)) (string, error) {
        fmt.Fprintln(w, "Creating shadow database...")
        shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
        if err != nil {
                return "", err
        }
        defer utils.DockerRemove(shadow)
        if err := start.WaitForHealthyService(ctx, start.HealthTimeout, shadow); err != nil {
                return "", err
        }
        if err := MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil {
                return "", err
        }
        shadowConfig := pgconn.Config{
                Host:     utils.Config.Hostname,
                Port:     utils.Config.Db.ShadowPort,
                User:     "postgres",
                Password: utils.Config.Db.Password,
                Database: "postgres",
        }
        if utils.IsLocalDatabase(config) {
                if declared, err := loadDeclaredSchemas(fsys); err != nil {
                        return "", err
                } else if len(declared) > 0 {
                        config = shadowConfig
                        config.Database = "contrib_regression"
                        if err := migrateBaseDatabase(ctx, config, declared, fsys, options...); err != nil {
                                return "", err
                        }
                }
        }
        // Load all user defined schemas
        if len(schema) == 0 {
                if schema, err = loadSchema(ctx, config, options...); err != nil {
                        return "", err
                }
        }
        fmt.Fprintln(w, "Diffing schemas:", strings.Join(schema, ","))
        source := utils.ToPostgresURL(shadowConfig)
        target := utils.ToPostgresURL(config)
        return differ(ctx, source, target, schema)
}

func migrateBaseDatabase(ctx context.Context, config pgconn.Config, migrations []string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
        fmt.Fprintln(os.Stderr, "Creating local database from declarative schemas:")
        msg := make([]string, len(migrations))
        for i, m := range migrations {
                msg[i] = fmt.Sprintf(" • %s", utils.Bold(m))
        }
        fmt.Fprintln(os.Stderr, strings.Join(msg, "\n"))
        conn, err := utils.ConnectLocalPostgres(ctx, config, options...)
        if err != nil {
                return err
        }
        defer conn.Close(context.Background())
        return migration.SeedGlobals(ctx, migrations, conn, afero.NewIOFS(fsys))
}

1	package diff
2
3	import (
4	"context"
5	_ "embed"
6	"fmt"
7	"io"
8	"io/fs"
9	"os"
10	"path/filepath"
11	"regexp"
12	"strconv"
13	"strings"
14	"time"
15
16	"github.com/cenkalti/backoff/v4"
17	"github.com/docker/docker/api/types/container"
18	"github.com/docker/docker/api/types/network"
19	"github.com/docker/go-connections/nat"
20	"github.com/go-errors/errors"
21	"github.com/jackc/pgconn"
22	"github.com/jackc/pgx/v4"
23	"github.com/spf13/afero"
24	"github.com/supabase/cli/internal/db/start"
25	"github.com/supabase/cli/internal/gen/keys"
26	"github.com/supabase/cli/internal/utils"
27	"github.com/supabase/cli/pkg/migration"
28	"github.com/supabase/cli/pkg/parser"
29	)
30
31	type DiffFunc func(context.Context, string, string, []string) (string, error)
32
33	func Run(ctx context.Context, schema []string, file string, config pgconn.Config, differ DiffFunc, fsys afero.Fs, confirmDrops bool, options ...func(*pgx.ConnConfig)) (err error) {	2✔
34	out, err := DiffDatabase(ctx, schema, config, os.Stderr, fsys, differ, options...)	2✔
35	if err != nil {	3✔
36	return err	1✔
37	}	1✔
38	branch := keys.GetGitBranch(fsys)	1✔
39	fmt.Fprintln(os.Stderr, "Finished "+utils.Aqua("supabase db diff")+" on branch "+utils.Aqua(branch)+".\n")	1✔
40		1✔
41	drops := findDropStatements(out)	1✔
42	if len(drops) > 0 {	1✔
NEW 43	if confirmDrops {	×
NEW 44	if err := showDropWarningAndConfirm(ctx, drops); err != nil {	×
NEW 45	return err	×
NEW 46	}	×
NEW 47	} else {	×
NEW 48	fmt.Fprintln(os.Stderr, "Found drop statements in schema diff. Please double check if these are expected:")	×
NEW 49	fmt.Fprintln(os.Stderr, utils.Yellow(strings.Join(drops, "\n")))	×
NEW 50	}	×
51	}
52
53	if err := SaveDiff(out, file, fsys); err != nil {	1✔
NEW 54	return err	×
55	}	×
56	return nil	1✔
57	}
58
59	func loadDeclaredSchemas(fsys afero.Fs) ([]string, error) {	1✔
60	if schemas := utils.Config.Db.Migrations.SchemaPaths; len(schemas) > 0 {	1✔
61	return schemas.Files(afero.NewIOFS(fsys))	×
62	}	×
63	if exists, err := afero.DirExists(fsys, utils.SchemasDir); err != nil {	1✔
64	return nil, errors.Errorf("failed to check schemas: %w", err)	×
65	} else if !exists {	1✔
66	return nil, nil	×
67	}	×
68	var declared []string	1✔
69	if err := afero.Walk(fsys, utils.SchemasDir, func(path string, info fs.FileInfo, err error) error {	10✔
70	if err != nil {	9✔
71	return err	×
72	}	×
73	if info.Mode().IsRegular() && filepath.Ext(info.Name()) == ".sql" {	13✔
74	declared = append(declared, path)	4✔
75	}	4✔
76	return nil	9✔
77	}); err != nil {	×
78	return nil, errors.Errorf("failed to walk dir: %w", err)	×
79	}	×
80	return declared, nil	1✔
81	}
82
83	// https://github.com/djrobstep/migra/blob/master/migra/statements.py#L6
84	var dropStatementPattern = regexp.MustCompile(`(?i)drop\s+`)
85
86	func findDropStatements(out string) []string {	2✔
87	lines, err := parser.SplitAndTrim(strings.NewReader(out))	2✔
88	if err != nil {	2✔
89	return nil	×
90	}	×
91	var drops []string	2✔
92	for _, line := range lines {	6✔
93	if dropStatementPattern.MatchString(line) {	6✔
94	drops = append(drops, line)	2✔
95	}	2✔
96	}
97	return drops	2✔
98	}
99
100	func showDropWarningAndConfirm(ctx context.Context, drops []string) error {	2✔
101	fmt.Fprintln(os.Stderr, utils.Red("⚠️ DANGEROUS OPERATION DETECTED"))	2✔
102	fmt.Fprintln(os.Stderr, utils.Red("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"))	2✔
103	fmt.Fprintln(os.Stderr, "")	2✔
104	fmt.Fprintln(os.Stderr, utils.Bold("The following DROP statements were found in your schema diff:"))	2✔
105	fmt.Fprintln(os.Stderr, "")	2✔
106	for _, drop := range drops {	4✔
107	fmt.Fprintln(os.Stderr, " "+utils.Red("▶ "+drop))	2✔
108	}	2✔
109	fmt.Fprintln(os.Stderr, "")	2✔
110	fmt.Fprintln(os.Stderr, utils.Yellow("❗ These operations may cause DATA LOSS:"))	2✔
111	fmt.Fprintln(os.Stderr, " • Column renames are detected as DROP + ADD, which will lose existing data")	2✔
112	fmt.Fprintln(os.Stderr, " • Table or schema deletions will permanently remove all data")	2✔
113	fmt.Fprintln(os.Stderr, " • Consider using RENAME operations instead of DROP + ADD for columns")	2✔
114	fmt.Fprintln(os.Stderr, "")	2✔
115	fmt.Fprintln(os.Stderr, utils.Bold("Please review the generated migration file carefully before proceeding."))	2✔
116	fmt.Fprintln(os.Stderr, "")	2✔
117		2✔
118	console := utils.NewConsole()	2✔
119	confirmed, err := console.PromptYesNo(ctx, "Do you want to continue with this potentially destructive operation?", false)	2✔
120	if err != nil {	2✔
NEW 121	return errors.Errorf("failed to get user confirmation: %w", err)	×
NEW 122	}	×
123	if !confirmed {	4✔
124	return errors.New("operation cancelled by user")	2✔
125	}	2✔
126
NEW 127	fmt.Fprintln(os.Stderr, "")	×
NEW 128	fmt.Fprintln(os.Stderr, utils.Yellow("⚠️ Proceeding with potentially destructive operation as requested."))	×
NEW 129	fmt.Fprintln(os.Stderr, "")	×
NEW 130	return nil	×
131	}
132
133	func loadSchema(ctx context.Context, config pgconn.Config, options ...func(*pgx.ConnConfig)) ([]string, error) {	×
134	conn, err := utils.ConnectByConfig(ctx, config, options...)	×
135	if err != nil {	×
136	return nil, err	×
137	}	×
138	defer conn.Close(context.Background())	×
139	// RLS policies in auth and storage schemas can be included with -s flag	×
140	return migration.ListUserSchemas(ctx, conn)	×
141	}
142
143	func CreateShadowDatabase(ctx context.Context, port uint16) (string, error) {	13✔
144	// Disable background workers in shadow database	13✔
145	config := start.NewContainerConfig("-c", "max_worker_processes=0")	13✔
146	hostPort := strconv.FormatUint(uint64(port), 10)	13✔
147	hostConfig := container.HostConfig{	13✔
148	PortBindings: nat.PortMap{"5432/tcp": []nat.PortBinding{{HostPort: hostPort}}},	13✔
149	AutoRemove: true,	13✔
150	}	13✔
151	networkingConfig := network.NetworkingConfig{}	13✔
152	if utils.Config.Db.MajorVersion <= 14 {	18✔
153	hostConfig.Tmpfs = map[string]string{"/docker-entrypoint-initdb.d": ""}	5✔
154	}	5✔
155	return utils.DockerStart(ctx, config, hostConfig, networkingConfig, "")	13✔
156	}
157
158	func ConnectShadowDatabase(ctx context.Context, timeout time.Duration, options ...func(pgx.ConnConfig)) (conn pgx.Conn, err error) {	9✔
159	// Retry until connected, cancelled, or timeout	9✔
160	policy := start.NewBackoffPolicy(ctx, timeout)	9✔
161	config := pgconn.Config{Port: utils.Config.Db.ShadowPort}	9✔
162	connect := func() (*pgx.Conn, error) {	18✔
163	return utils.ConnectLocalPostgres(ctx, config, options...)	9✔
164	}	9✔
165	return backoff.RetryWithData(connect, policy)	9✔
166	}
167
168	// Required to bypass pg_cron check: https://github.com/citusdata/pg_cron/blob/main/pg_cron.sql#L3
169	const CREATE_TEMPLATE = "CREATE DATABASE contrib_regression TEMPLATE postgres"
170
171	func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {	7✔
172	migrations, err := migration.ListLocalMigrations(utils.MigrationsDir, afero.NewIOFS(fsys))	7✔
173	if err != nil {	8✔
174	return err	1✔
175	}	1✔
176	conn, err := ConnectShadowDatabase(ctx, 10*time.Second, options...)	6✔
177	if err != nil {	7✔
178	return err	1✔
179	}	1✔
180	defer conn.Close(context.Background())	5✔
181	if err := start.SetupDatabase(ctx, conn, container[:12], os.Stderr, fsys); err != nil {	7✔
182	return err	2✔
183	}	2✔
184	if _, err := conn.Exec(ctx, CREATE_TEMPLATE); err != nil {	3✔
185	return errors.Errorf("failed to create template database: %w", err)	×
186	}	×
187	return migration.ApplyMigrations(ctx, migrations, conn, afero.NewIOFS(fsys))	3✔
188	}
189
190	func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ func(context.Context, string, string, []string) (string, error), options ...func(*pgx.ConnConfig)) (string, error) {	7✔
191	fmt.Fprintln(w, "Creating shadow database...")	7✔
192	shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)	7✔
193	if err != nil {	10✔
194	return "", err	3✔
195	}	3✔
196	defer utils.DockerRemove(shadow)	4✔
197	if err := start.WaitForHealthyService(ctx, start.HealthTimeout, shadow); err != nil {	5✔
198	return "", err	1✔
199	}	1✔
200	if err := MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil {	4✔
201	return "", err	1✔
202	}	1✔
203	shadowConfig := pgconn.Config{	2✔
204	Host: utils.Config.Hostname,	2✔
205	Port: utils.Config.Db.ShadowPort,	2✔
206	User: "postgres",	2✔
207	Password: utils.Config.Db.Password,	2✔
208	Database: "postgres",	2✔
209	}	2✔
210	if utils.IsLocalDatabase(config) {	2✔
211	if declared, err := loadDeclaredSchemas(fsys); err != nil {	×
212	return "", err	×
213	} else if len(declared) > 0 {	×
214	config = shadowConfig	×
215	config.Database = "contrib_regression"	×
216	if err := migrateBaseDatabase(ctx, config, declared, fsys, options...); err != nil {	×
217	return "", err	×
218	}	×
219	}
220	}
221	// Load all user defined schemas
222	if len(schema) == 0 {	2✔
223	if schema, err = loadSchema(ctx, config, options...); err != nil {	×
224	return "", err	×
225	}	×
226	}
227	fmt.Fprintln(w, "Diffing schemas:", strings.Join(schema, ","))	2✔
228	source := utils.ToPostgresURL(shadowConfig)	2✔
229	target := utils.ToPostgresURL(config)	2✔
230	return differ(ctx, source, target, schema)	2✔
231	}
232
233	func migrateBaseDatabase(ctx context.Context, config pgconn.Config, migrations []string, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {	×
234	fmt.Fprintln(os.Stderr, "Creating local database from declarative schemas:")	×
235	msg := make([]string, len(migrations))	×
236	for i, m := range migrations {	×
237	msg[i] = fmt.Sprintf(" • %s", utils.Bold(m))	×
238	}	×
239	fmt.Fprintln(os.Stderr, strings.Join(msg, "\n"))	×
240	conn, err := utils.ConnectLocalPostgres(ctx, config, options...)	×
241	if err != nil {	×
242	return err	×
243	}	×
244	defer conn.Close(context.Background())	×
245	return migration.SeedGlobals(ctx, migrations, conn, afero.NewIOFS(fsys))	×
246	}

supabase / cli / 16535713976

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous