8392258799

Committed 22 Mar 2024 02:58PM UTC coverage: 58.963% (-0.06%) from 59.021%

Build # 8392258799

Build Type

Pull #2086

github

Committed by

web-flow

Commit Message

Merge branch 'develop' into inc-pub

Pull Request Pull Request #2086: fix: include publication in db dump

Run Details

7 of 13 new or added lines in 2 files covered. (53.85%)

5 existing lines in 1 file now uncovered.

6263 of 10622 relevant lines covered (58.96%)

688.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

42.96

/internal/db/dump/dump.go

package dump

import (
        "context"
        _ "embed"
        "fmt"
        "io"
        "os"
        "strings"

        "github.com/docker/docker/api/types/container"
        "github.com/docker/docker/api/types/network"
        "github.com/go-errors/errors"
        "github.com/jackc/pgconn"
        "github.com/spf13/afero"
        "github.com/supabase/cli/internal/utils"
)

var (
        //go:embed templates/dump_schema.sh
        dumpSchemaScript string
        //go:embed templates/dump_data.sh
        dumpDataScript string
        //go:embed templates/dump_role.sh
        dumpRoleScript string
)

func Run(ctx context.Context, path string, config pgconn.Config, schema, excludeTable []string, dataOnly, roleOnly, keepComments, useCopy, dryRun bool, fsys afero.Fs) error {
        // Initialize output stream
        var outStream afero.File
        if len(path) > 0 {
                f, err := fsys.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
                if err != nil {
                        return errors.Errorf("failed to open dump file: %w", err)
                }
                defer f.Close()
                outStream = f
        } else {
                outStream = os.Stdout
        }
        // Load the requested script
        if dryRun {
                fmt.Fprintln(os.Stderr, "DRY RUN: *only* printing the pg_dump script to console.")
        }
        db := "remote"
        if utils.IsLocalDatabase(config) {
                db = "local"
        }
        if dataOnly {
                fmt.Fprintf(os.Stderr, "Dumping data from %s database...\n", db)
                return dumpData(ctx, config, schema, excludeTable, useCopy, dryRun, outStream)
        } else if roleOnly {
                fmt.Fprintf(os.Stderr, "Dumping roles from %s database...\n", db)
                return dumpRole(ctx, config, keepComments, dryRun, outStream)
        }
        fmt.Fprintf(os.Stderr, "Dumping schemas from %s database...\n", db)
        return DumpSchema(ctx, config, schema, keepComments, dryRun, outStream)
}

func DumpSchema(ctx context.Context, config pgconn.Config, schema []string, keepComments, dryRun bool, stdout io.Writer) error {
        var env []string
        if len(schema) > 0 {
                // Must append flag because empty string results in error
                env = append(env, "EXTRA_FLAGS=--schema="+strings.Join(schema, "|"))
        } else {
                env = append(env, "EXCLUDED_SCHEMAS="+strings.Join(utils.InternalSchemas, "|"))
        }
        if !keepComments {
                env = append(env, "EXTRA_SED=/^--/d")
        }
        return dump(ctx, config, dumpSchemaScript, env, dryRun, stdout)
}

func dumpData(ctx context.Context, config pgconn.Config, schema, excludeTable []string, useCopy, dryRun bool, stdout io.Writer) error {
        // We want to dump user data in auth, storage, etc. for migrating to new project
        excludedSchemas := []string{
                "information_schema",
                "pg_*", // Wildcard pattern follows pg_dump
                // Owned by extensions
                // "cron",
                "graphql",
                "graphql_public",
                // "net",
                // "pgsodium",
                // "pgsodium_masks",
                "pgtle",
                "repack",
                "tiger",
                "tiger_data",
                "timescaledb_*",
                "_timescaledb_*",
                "topology",
                // "vault",
                // Managed by Supabase
                // "auth",
                "extensions",
                "pgbouncer",
                "realtime",
                "_realtime",
                // "storage",
                "_analytics",
                // "supabase_functions",
                "supabase_migrations",
        }
        var env []string
        if len(schema) > 0 {
                env = append(env, "INCLUDED_SCHEMAS="+strings.Join(schema, "|"))
        } else {
                env = append(env, "INCLUDED_SCHEMAS=*", "EXCLUDED_SCHEMAS="+strings.Join(excludedSchemas, "|"))
        }
        var extraFlags []string
        if !useCopy {
                extraFlags = append(extraFlags, "--column-inserts", "--rows-per-insert 100000")
        }
        for _, table := range excludeTable {
                extraFlags = append(extraFlags, "--exclude-table "+table)
        }
        if len(extraFlags) > 0 {
                env = append(env, "EXTRA_FLAGS="+strings.Join(extraFlags, " "))
        }
        return dump(ctx, config, dumpDataScript, env, dryRun, stdout)
}

func dumpRole(ctx context.Context, config pgconn.Config, keepComments, dryRun bool, stdout io.Writer) error {
        env := []string{}
        if !keepComments {
                env = append(env, "EXTRA_SED=/^--/d")
        }
        return dump(ctx, config, dumpRoleScript, env, dryRun, stdout)
}

func dump(ctx context.Context, config pgconn.Config, script string, env []string, dryRun bool, stdout io.Writer) error {
        allEnvs := append(env,
                "PGHOST="+config.Host,
                fmt.Sprintf("PGPORT=%d", config.Port),
                "PGUSER="+config.User,
                "PGPASSWORD="+config.Password,
                "PGDATABASE="+config.Database,
                "RESERVED_ROLES="+strings.Join(utils.ReservedRoles, "|"),
                "ALLOWED_CONFIGS="+strings.Join(utils.AllowedConfigs, "|"),
        )
        if dryRun {
                envMap := make(map[string]string, len(allEnvs))
                for _, e := range allEnvs {
                        index := strings.IndexByte(e, '=')
                        if index < 0 {
                                continue
                        }
                        envMap[e[:index]] = e[index+1:]
                }
                expanded := os.Expand(script, func(key string) string {
                        // Bash variable expansion is unsupported:
                        // https://github.com/golang/go/issues/47187
                        parts := strings.Split(key, ":")
                        return envMap[parts[0]]
                })
                fmt.Println(expanded)
                return nil
        }
        return utils.DockerRunOnceWithConfig(
                ctx,
                container.Config{
                        Image: utils.Pg15Image,
                        Env:   allEnvs,
                        Cmd:   []string{"bash", "-c", script, "--"},
                },
                container.HostConfig{
                        NetworkMode: container.NetworkMode("host"),
                },
                network.NetworkingConfig{},
                "",
                stdout,
                os.Stderr,
        )
}

1	package dump
2
3	import (
4	"context"
5	_ "embed"
6	"fmt"
7	"io"
8	"os"
9	"strings"
10
11	"github.com/docker/docker/api/types/container"
12	"github.com/docker/docker/api/types/network"
13	"github.com/go-errors/errors"
14	"github.com/jackc/pgconn"
15	"github.com/spf13/afero"
16	"github.com/supabase/cli/internal/utils"
17	)
18
19	var (
20	//go:embed templates/dump_schema.sh
21	dumpSchemaScript string
22	//go:embed templates/dump_data.sh
23	dumpDataScript string
24	//go:embed templates/dump_role.sh
25	dumpRoleScript string
26	)
27
28	func Run(ctx context.Context, path string, config pgconn.Config, schema, excludeTable []string, dataOnly, roleOnly, keepComments, useCopy, dryRun bool, fsys afero.Fs) error {	4✔
29	// Initialize output stream	4✔
30	var outStream afero.File	4✔
31	if len(path) > 0 {	6✔
32	f, err := fsys.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_TRUNC, 0644)	2✔
33	if err != nil {	3✔
34	return errors.Errorf("failed to open dump file: %w", err)	1✔
35	}	1✔
36	defer f.Close()	1✔
37	outStream = f	1✔
38	} else {	2✔
39	outStream = os.Stdout	2✔
40	}	2✔
41	// Load the requested script
42	if dryRun {	3✔
43	fmt.Fprintln(os.Stderr, "DRY RUN: only printing the pg_dump script to console.")	×
44	}	×
45	db := "remote"	3✔
46	if utils.IsLocalDatabase(config) {	3✔
NEW 47	db = "local"	×
NEW 48	}	×
49	if dataOnly {	3✔
NEW 50	fmt.Fprintf(os.Stderr, "Dumping data from %s database...\n", db)	×
51	return dumpData(ctx, config, schema, excludeTable, useCopy, dryRun, outStream)	×
52	} else if roleOnly {	3✔
NEW 53	fmt.Fprintf(os.Stderr, "Dumping roles from %s database...\n", db)	×
54	return dumpRole(ctx, config, keepComments, dryRun, outStream)	×
55	}	×
56	fmt.Fprintf(os.Stderr, "Dumping schemas from %s database...\n", db)	3✔
57	return DumpSchema(ctx, config, schema, keepComments, dryRun, outStream)	3✔
58	}
59
60	func DumpSchema(ctx context.Context, config pgconn.Config, schema []string, keepComments, dryRun bool, stdout io.Writer) error {	9✔
61	var env []string	9✔
62	if len(schema) > 0 {	14✔
63	// Must append flag because empty string results in error	5✔
64	env = append(env, "EXTRA_FLAGS=--schema="+strings.Join(schema, "\|"))	5✔
65	} else {	9✔
66	env = append(env, "EXCLUDED_SCHEMAS="+strings.Join(utils.InternalSchemas, "\|"))	4✔
67	}	4✔
68	if !keepComments {	18✔
69	env = append(env, "EXTRA_SED=/^--/d")	9✔
70	}	9✔
71	return dump(ctx, config, dumpSchemaScript, env, dryRun, stdout)	9✔
72	}
73
74	func dumpData(ctx context.Context, config pgconn.Config, schema, excludeTable []string, useCopy, dryRun bool, stdout io.Writer) error {	×
75	// We want to dump user data in auth, storage, etc. for migrating to new project	×
76	excludedSchemas := []string{	×
77	"information_schema",	×
78	"pg_*", // Wildcard pattern follows pg_dump	×
79	// Owned by extensions	×
80	// "cron",	×
81	"graphql",	×
82	"graphql_public",	×
83	// "net",	×
84	// "pgsodium",	×
85	// "pgsodium_masks",	×
86	"pgtle",	×
87	"repack",	×
88	"tiger",	×
89	"tiger_data",	×
90	"timescaledb_*",	×
91	"_timescaledb_*",	×
92	"topology",	×
93	// "vault",	×
94	// Managed by Supabase	×
95	// "auth",	×
96	"extensions",	×
97	"pgbouncer",	×
98	"realtime",	×
99	"_realtime",	×
100	// "storage",	×
101	"_analytics",	×
102	// "supabase_functions",	×
103	"supabase_migrations",	×
104	}	×
105	var env []string	×
106	if len(schema) > 0 {	×
107	env = append(env, "INCLUDED_SCHEMAS="+strings.Join(schema, "\|"))	×
108	} else {	×
109	env = append(env, "INCLUDED_SCHEMAS=*", "EXCLUDED_SCHEMAS="+strings.Join(excludedSchemas, "\|"))	×
110	}	×
111	var extraFlags []string	×
112	if !useCopy {	×
113	extraFlags = append(extraFlags, "--column-inserts", "--rows-per-insert 100000")	×
114	}	×
115	for _, table := range excludeTable {	×
116	extraFlags = append(extraFlags, "--exclude-table "+table)	×
117	}	×
118	if len(extraFlags) > 0 {	×
119	env = append(env, "EXTRA_FLAGS="+strings.Join(extraFlags, " "))	×
120	}	×
121	return dump(ctx, config, dumpDataScript, env, dryRun, stdout)	×
122	}
123
124	func dumpRole(ctx context.Context, config pgconn.Config, keepComments, dryRun bool, stdout io.Writer) error {	×
125	env := []string{}	×
126	if !keepComments {	×
127	env = append(env, "EXTRA_SED=/^--/d")	×
128	}	×
129	return dump(ctx, config, dumpRoleScript, env, dryRun, stdout)	×
130	}
131
132	func dump(ctx context.Context, config pgconn.Config, script string, env []string, dryRun bool, stdout io.Writer) error {	9✔
133	allEnvs := append(env,	9✔
134	"PGHOST="+config.Host,	9✔
135	fmt.Sprintf("PGPORT=%d", config.Port),	9✔
136	"PGUSER="+config.User,	9✔
137	"PGPASSWORD="+config.Password,	9✔
138	"PGDATABASE="+config.Database,	9✔
139	"RESERVED_ROLES="+strings.Join(utils.ReservedRoles, "\|"),	9✔
140	"ALLOWED_CONFIGS="+strings.Join(utils.AllowedConfigs, "\|"),	9✔
141	)	9✔
142	if dryRun {	9✔
143	envMap := make(map[string]string, len(allEnvs))	×
144	for _, e := range allEnvs {	×
145	index := strings.IndexByte(e, '=')	×
146	if index < 0 {	×
147	continue	×
148	}
149	envMap[e[:index]] = e[index+1:]	×
150	}
151	expanded := os.Expand(script, func(key string) string {	×
152	// Bash variable expansion is unsupported:	×
153	// https://github.com/golang/go/issues/47187	×
154	parts := strings.Split(key, ":")	×
155	return envMap[parts[0]]	×
156	})	×
157	fmt.Println(expanded)	×
158	return nil	×
159	}
160	return utils.DockerRunOnceWithConfig(	9✔
161	ctx,	9✔
162	container.Config{	9✔
163	Image: utils.Pg15Image,	9✔
164	Env: allEnvs,	9✔
165	Cmd: []string{"bash", "-c", script, "--"},	9✔
166	},	9✔
167	container.HostConfig{	9✔
168	NetworkMode: container.NetworkMode("host"),	9✔
169	},	9✔
170	network.NetworkingConfig{},	9✔
171	"",	9✔
172	stdout,	9✔
173	os.Stderr,	9✔
174	)	9✔
175	}

supabase / cli / 8392258799

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous