25829447167

Committed 13 May 2026 09:55PM UTC coverage: 92.491% (-0.04%) from 92.527%

Build # 25829447167

Build Type

push

github

Committed by

evanelias

Commit Message

host option: support external command shellouts directly

Skeema has always supported use of external commands for obtaining hostnames
via the host-wrapper option, which is designed to be configured a single time
generically in a top-level directory. However, that option requires that the
host option is still set in the appropriate subdir .skeema files -- even
if just set to a dummy/placeholder value, so that Skeema still knows which
dirs are supposed to map to hosts.

That host-wrapper setup can be unnecessarily confusing for cases where it is
more desirable to define the external command directly in each .skeema file
that should map to hosts, or in cases where there is only a single host-level
directory anyway.

This commit augments the host option accordingly, to provide a simpler
alternative to host-wrapper for these situations. Now, any value wrapped in
`backticks` is treated as an external shell command to execute. The command's
STDOUT is captured for use as the list of hostnames, just like host-wrapper's
functionality, but without needing to configure that separate option.

In the backtick-wrapped value, the following placeholder variables are
supported for dynamic interpolation:

  {ENVIRONMENT}: The environment name from the command-line
  {DIRNAME}:     Base name (last path component) of the dir being processed
  {DIRPATH}:     Full path of the dir being processed
  {SCHEMA}:      Schema name being processed, if constant or $ENV

To be clear, this commit does NOT deprecate the host-wrapper option, nor fully
replace all of its use-cases. In situations where a common service-discovery
program should be configured for schema repo containing many distinct database
clusters, the best approach for that situation is to stay with host-wrapper in
a top-level .skeema option file, and subdirs that define host as service-
discovery lookup keys.

In contrast, use of the new host=`...` functionality makes more sense in cases
where each subdir uses a diffe... (continued)

Coverage Stats

38 of 43 new or added lines in 3 files covered. (88.37%)

2 existing lines in 1 file now uncovered.

10962 of 11852 relevant lines covered (92.49%)

1.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.19

/cmd_pull.go

package main

import (
        "database/sql"
        "fmt"
        "os"

        log "github.com/sirupsen/logrus"
        "github.com/skeema/mybase"
        "github.com/skeema/skeema/internal/dumper"
        "github.com/skeema/skeema/internal/fs"
        "github.com/skeema/skeema/internal/tengo"
        "github.com/skeema/skeema/internal/workspace"
)

func init() {
        summary := "Update the filesystem representation of schemas"
        desc := "Updates the existing filesystem representation of the schemas on a DB " +
                "server. Use this command when changes have been applied to the database " +
                "manually or outside of Skeema, in order to make the filesystem representation " +
                "reflect those changes.\n\n" +
                "You may optionally pass an environment name as a command-line arg. This will affect " +
                "which section of .skeema config files is used for processing. For example, " +
                "running `skeema pull staging` will apply config directives from the " +
                "[staging] section of config files, as well as any sectionless directives at the " +
                "top of the file. If no environment name is supplied, the default is " +
                "\"production\"."

        cmd := mybase.NewCommand("pull", summary, desc, PullHandler)
        cmd.AddOption(mybase.BoolOption("include-auto-inc", 0, false, "Include starting auto-inc values in new table files, and update in existing files"))
        cmd.AddOption(mybase.BoolOption("format", 0, true, "Reformat SQL statements to match canonical SHOW CREATE"))
        cmd.AddOption(mybase.BoolOption("normalize", 0, true, "(deprecated alias for format)").Hidden().MarkDeprecated("This option will be removed in Skeema v2. Use the equivalent --format option instead, which currently defaults to true, but will default to false in Skeema v2."))
        cmd.AddOption(mybase.BoolOption("new-schemas", 0, true, "Detect any new schemas and populate new dirs for them"))
        cmd.AddOption(mybase.BoolOption("update-partitioning", 0, false, "Update PARTITION BY clauses in existing table files"))
        cmd.AddOption(mybase.BoolOption("strip-partitioning", 0, false, "Omit PARTITION BY clause when writing partitioned tables to filesystem"))
        workspace.AddCommandOptions(cmd)
        cmd.AddArg("environment", "production", false)
        CommandSuite.AddSubCommand(cmd)
}

// PullHandler is the handler method for `skeema pull`
func PullHandler(cfg *mybase.Config) error {
        dir, err := fs.ParseDir(".", cfg)
        if err != nil {
                return err
        }
        if err := dir.CheckGenerator(generatorString()); err != nil {
                return err
        }
        if !dir.Config.Supplied("format") && !dir.Config.Supplied("normalize") {
                log.Debug("Upgrade notice: the --format option, which currently defaults to true in Skeema v1, will change to default to false in Skeema v2. For more information, visit https://www.skeema.io/v2-changes")
        }

        // pullWalker returns the "worst" (highest) exit code it encounters. We care
        // about the exit code, but not the error message, since any error will already
        // have been logged. (Multiple errors may have been encountered along the way,
        // and it's simpler to log them when they occur, rather than needlessly
        // collecting them.)
        err = pullWalker(dir, 5)
        return NewExitValue(ExitCode(err), "")
}

func pullWalker(dir *fs.Dir, maxDepth int) error {
        if dir.ParseError != nil {
                log.Warnf("Skipping %s: %s", dir, dir.ParseError)
                return NewExitValue(CodeBadConfig, "")
        }

        var instance *tengo.Instance
        var err error
        if dir.Config.Changed("host") {
                instance, err = dir.FirstInstance()
                if err != nil {
                        log.Warnf("Skipping %s: %s", dir, err)
                        return NewExitValue(CodePartialError, "")
                }
        }

        // dir defines a schema in .skeema, and/or has *.sql files
        if dir.HasSchema() {
                // If we cannot pull due to lack of an instance, make it clear to the user
                if instance == nil {
                        log.Errorf("Skipping %s: No host defined for environment %q", dir, dir.Config.Get("environment"))
                        if dir.Config.Changed("host-wrapper") {
                                log.Error("The host-wrapper option controls which hosts this directory maps to, but the executed command did not output any hostnames.")
                        } else if rawHost := dir.Config.GetRaw("host"); rawHost != "" && rawHost[0] == '`' {
                                log.Error("The host option is set to a backtick-wrapped external command, but that command did not output any hostnames.")
                        } else {
                                log.Errorf("This command requires a hostname to pull from, which can be specified using the host option in a [%s] section of the .skeema file in this directory, or in a parent directory.", dir.Config.Get("environment"))
                        }
                        return NewExitValue(CodeBadConfig, "")
                }

                // Otherwise, we're in a "flat" dir that defines both host and schema: update
                // the flavor if needed and process the pull operation on *.sql files, but no
                // need to look for new schemas with this layout
                updateFlavor(dir, instance)
                updateGenerator(dir)
                _, err := pullSchemaDir(dir, instance) // already logs err (if non-nil)
                return err
        }

        subdirs, err := dir.Subdirs()
        if err != nil {
                log.Errorf("Cannot list subdirs of %s: %s", dir, err)
                return NewExitValue(CodePartialError, "")
        } else if len(subdirs) > 0 && maxDepth <= 0 {
                log.Errorf("Not walking subdirs of %s: max depth reached", dir)
                return NewExitValue(CodePartialError, "")
        }

        allSchemaNames := []string{}
        for _, sub := range subdirs {
                // If dir does not define host, simply recurse into subdirs.
                if instance == nil {
                        subErr := pullWalker(sub, maxDepth-1)
                        err = HighestExitCode(err, subErr)
                        continue
                }

                // Otherwise, dir defines host but not schema. Treat subdirs as schema dirs,
                // and use the combined list of handled schemas to figure out whether any
                // new schema dirs need to be created (if requested).
                subSchemaNames, subErr := pullSchemaDir(sub, instance) // already logs subErr (if non-nil)
                err = HighestExitCode(err, subErr)
                allSchemaNames = append(allSchemaNames, subSchemaNames...)
        }

        if instance != nil {
                updateFlavor(dir, instance)
                updateGenerator(dir)
                if dir.Config.GetBool("new-schemas") && err == nil {
                        if err = findNewSchemas(dir, instance, allSchemaNames); err != nil {
                                log.Warnf("Unable to populate new schemas from %s: %s", dir, err)
                                return NewExitValue(CodePartialError, "")
                        }
                }
        }
        return err
}

// pullSchemaDir updates all logical schemas in dir to reflect the actual
// definitions found in instance. A slice of handled schema names is returned,
// along with any error encountered.
func pullSchemaDir(dir *fs.Dir, instance *tengo.Instance) (schemaNames []string, err error) {
        if dir.ParseError != nil {
                log.Warnf("Skipping %s: %s", dir, dir.ParseError)
                return nil, NewExitValue(CodePartialError, "")
        }
        if len(dir.LogicalSchemas) > 0 {
                // TODO: support multiple logical schemas per dir
                logicalSchema := dir.LogicalSchemas[0]
                schemaNames, err = pullLogicalSchema(dir, instance, logicalSchema)
                if err != nil {
                        log.Errorf("Skipping %s: %s\n", dir, err)
                }
        }
        return
}

// pullLogicalSchema performs appropriate pull logic on a dir that maps to one or
// more schemas. A slice of handled schema names is returned, along with any
// error encountered.
func pullLogicalSchema(dir *fs.Dir, instance *tengo.Instance, logicalSchema *fs.LogicalSchema) (schemaNames []string, err error) {
        // With non-zero lower_case_table_names, force names to lowercase as needed in
        // logicalSchema, so that statements can be correctly linked to objects
        if lctn := instance.NameCaseMode(); lctn > tengo.NameCaseAsIs {
                if err := logicalSchema.LowerCaseNames(lctn); err != nil {
                        return nil, err
                }
        }

        if logicalSchema.Name != "" {
                schemaNames = []string{logicalSchema.Name}
        } else if schemaNames, err = dir.SchemaNames(instance); err != nil {
                return nil, fmt.Errorf("unable to fetch schema names mapped by this dir: %w", err)
        }
        if len(schemaNames) == 0 {
                log.Warnf("Ignoring directory %s -- did not map to any schema names for environment %q\n", dir, dir.Config.Get("environment"))
                return
        }
        instSchema, err := instance.Schema(schemaNames[0])
        if err == sql.ErrNoRows {
                log.Infof("Deleted directory %s -- schema %s no longer exists\n", dir, schemaNames[0])
                return nil, dir.Delete()
        } else if err != nil {
                return nil, fmt.Errorf("Unable to fetch schema %s from %s: %s", schemaNames[0], instance, err)
        }
        instSchema.StripMatches(dir.IgnorePatterns)

        log.Infof("Updating %s to reflect %s %s", dir, instance, instSchema.Name)

        // Handle changes in schema's default character set and/or collation by
        // persisting changes to the dir's option file.
        if err := updateCharSetCollation(dir, instSchema); err != nil {
                return nil, err
        }

        dumpOpts := dumper.Options{
                IncludeAutoInc: dir.Config.GetBool("include-auto-inc"),
        }
        if !dir.Config.GetBool("update-partitioning") {
                if dir.Config.GetBool("strip-partitioning") {
                        // Undocumented due to potential confusion, but supported just like in init
                        dumpOpts.Partitioning = tengo.PartitioningRemove
                } else {
                        // Without --update-partitioning, retain whatever partitioning clause (or
                        // lack of clause) was already present in the *.sql files for existing tables.
                        dumpOpts.Partitioning = tengo.PartitioningKeep
                }
        }

        // When --skip-format is in use, we only want to update objects that have
        // actual functional modifications, NOT just cosmetic/formatting differences.
        // To make this distinction, we need to actually execute the *.sql files in a
        // Workspace and run a diff against it.
        if !dir.Config.GetBool("format") || !dir.Config.GetBool("normalize") {
                mods := statementModifiersForPull(dir.Config, instance)
                opts, err := workspace.OptionsForDir(dir, instance)
                if err != nil {
                        return nil, WrapExitCode(CodeBadConfig, err)
                }
                inDiff, err := objectsInDiff(logicalSchema, instSchema, opts, mods)
                if err != nil {
                        return nil, err
                }
                dumpOpts.OnlyKeys(inDiff)
        }

        _, err = dumper.DumpSchema(instSchema, dir, dumpOpts)
        if err == nil {
                os.Stderr.WriteString("\n")
        }
        return
}

func statementModifiersForPull(config *mybase.Config, instance *tengo.Instance) tengo.StatementModifiers {
        // We're permissive of unsafe operations here since we don't ever actually
        // execute the generated statement! We just examine its type.
        mods := tengo.StatementModifiers{
                AllowUnsafe: true,
        }
        // pull command updates next auto-increment value for existing table always
        // if requested, or only if previously present in file otherwise
        if config.GetBool("include-auto-inc") {
                mods.NextAutoInc = tengo.NextAutoIncAlways
        } else {
                mods.NextAutoInc = tengo.NextAutoIncIfAlready
        }
        instFlavor, confFlavor := instance.Flavor(), tengo.ParseFlavor(config.Get("flavor"))
        if !instFlavor.Known() && confFlavor.Known() {
                mods.Flavor = confFlavor
        } else {
                mods.Flavor = instFlavor
        }
        // Unless user specifically wants to update partitioning clauses, apply a
        // statement modifier to make some partitioning-related AlterClause types
        // return an empty statement, to exclude them from being rewritten if their
        // only differences are partitioning-related.
        if !config.GetBool("update-partitioning") {
                mods.Partitioning = tengo.PartitioningKeep
        }
        return mods
}

// objectsInDiff returns a map whose keys are tengo.ObjectKeys of objects that
// have modifications in instSchema that aren't reflected in their filesystem
// representation yet. This also includes objects whose filesystem Statement has
// a SQL syntax error. The return value does not include tables whose
// differences are cosmetic / formatting-related, or are otherwise ignored by
// mods.
func objectsInDiff(logicalSchema *fs.LogicalSchema, instSchema *tengo.Schema, opts workspace.Options, mods tengo.StatementModifiers) ([]tengo.ObjectKey, error) {
        wsSchema, err := workspace.ExecLogicalSchema(logicalSchema, opts)
        if err != nil {
                return nil, fmt.Errorf("Error introspecting filesystem version of schema %s: %s", instSchema.Name, err)
        }
        log.Debugf("Workspace performance using %s:\n%s", wsSchema.Info, wsSchema.Timers)

        // Run a diff, and create a map to track objects in the diff
        diff := tengo.NewSchemaDiff(wsSchema.Schema, instSchema)
        inDiff := make([]tengo.ObjectKey, 0)
        for _, od := range diff.ObjectDiffs() {
                odStatement, odStatementErr := od.Statement(mods)
                key := od.ObjectKey()
                // Errors are fatal, except for UnsupportedDiffError which we can safely
                // ignore (since pull doesn't actually run ALTERs; it just needs to know
                // what was altered)
                if odStatementErr != nil && !tengo.IsUnsupportedDiff(odStatementErr) {
                        return nil, odStatementErr
                }
                // mods may cause the diff to be a no-op; only include it in result if this
                // isn't the case
                if odStatement != "" {
                        inDiff = append(inDiff, key)
                }
        }

        // Treat objects with syntax errors as modified, since it isn't possible for
        // the filesystem definition to match the live definition in this case.
        inDiff = append(inDiff, wsSchema.FailedKeys()...)

        return inDiff, nil
}

// updateFlavor updates the dir's .skeema option file if the instance's current
// flavor does not match what's in the file. However, it leaves the value in the
// file alone if it's specified and we're unable to detect the instance's
// vendor, as this gives operators the ability to manually override an
// undetectable flavor.
func updateFlavor(dir *fs.Dir, instance *tengo.Instance) {
        instFlavor := instance.Flavor()
        if !instFlavor.Known() || instFlavor.Family().String() == dir.Config.Get("flavor") {
                return
        }
        dir.OptionFile.SetOptionValue(dir.Config.Get("environment"), "flavor", instFlavor.Family().String())
        if err := dir.OptionFile.Write(true); err != nil {
                log.Warnf("Unable to update flavor in %s: %s", dir.OptionFile.Path(), err)
        } else {
                log.Infof("Wrote %s -- updated flavor to %s", dir.OptionFile.Path(), instFlavor.Family().String())
        }
}

func updateGenerator(dir *fs.Dir) {
        currentGenerator := generatorString()
        if dir.Config.Get("generator") == currentGenerator {
                return
        }
        dir.OptionFile.SetOptionValue("", "generator", currentGenerator)
        // ignoring errors; failure not terribly important
        if err := dir.OptionFile.Write(true); err == nil {
                log.Infof("Wrote %s -- updated generator to %s", dir.OptionFile.Path(), currentGenerator)
        }
}

// updateCharSetCollation updates the dir's .skeema option file if the schema's
// current default charset or collation does not match what's in the file.
func updateCharSetCollation(dir *fs.Dir, instSchema *tengo.Schema) error {
        if dir.Config.Get("default-character-set") != instSchema.CharSet || dir.Config.Get("default-collation") != instSchema.Collation {
                dir.OptionFile.SetOptionValue("", "default-character-set", instSchema.CharSet)
                dir.OptionFile.SetOptionValue("", "default-collation", instSchema.Collation)
                if err := dir.OptionFile.Write(true); err != nil {
                        return fmt.Errorf("Unable to update character set and collation for %s: %s", dir.OptionFile.Path(), err)
                }
                log.Infof("Wrote %s -- updated schema-level default-character-set and default-collation", dir.OptionFile.Path())
        }
        return nil
}

func findNewSchemas(dir *fs.Dir, instance *tengo.Instance, seenNames []string) error {
        subdirHasSchema := make(map[string]bool)
        for _, name := range seenNames {
                subdirHasSchema[name] = true
        }

        schemaNames, err := instance.SchemaNames()
        if err != nil {
                return err
        }
        alreadyWarned := dir.Config.Supplied("new-schemas") // no need to warn if option set explicitly
        for _, name := range schemaNames {
                // If no existing subdir maps to the schema, we need to create and populate new dir
                if !subdirHasSchema[name] {
                        if !alreadyWarned {
                                alreadyWarned = true
                                log.Debug("Upgrade notice: the --new-schemas option, which currently defaults to true in Skeema v1, will change to default to false in Skeema v2. For more information, visit https://www.skeema.io/v2-changes")
                        }
                        s, err := instance.Schema(name)
                        if err != nil {
                                return err
                        }
                        s.StripMatches(dir.IgnorePatterns)
                        // use same logic from init command
                        if err := PopulateSchemaDir(s, dir, true); err != nil {
                                return err
                        }
                }
        }

        return nil
}

1	package main
2
3	import (
4	"database/sql"
5	"fmt"
6	"os"
7
8	log "github.com/sirupsen/logrus"
9	"github.com/skeema/mybase"
10	"github.com/skeema/skeema/internal/dumper"
11	"github.com/skeema/skeema/internal/fs"
12	"github.com/skeema/skeema/internal/tengo"
13	"github.com/skeema/skeema/internal/workspace"
14	)
15
16	func init() {	1✔
17	summary := "Update the filesystem representation of schemas"	1✔
18	desc := "Updates the existing filesystem representation of the schemas on a DB " +	1✔
19	"server. Use this command when changes have been applied to the database " +	1✔
20	"manually or outside of Skeema, in order to make the filesystem representation " +	1✔
21	"reflect those changes.\n\n" +	1✔
22	"You may optionally pass an environment name as a command-line arg. This will affect " +	1✔
23	"which section of .skeema config files is used for processing. For example, " +	1✔
24	"running `skeema pull staging` will apply config directives from the " +	1✔
25	"[staging] section of config files, as well as any sectionless directives at the " +	1✔
26	"top of the file. If no environment name is supplied, the default is " +	1✔
27	"\"production\"."	1✔
28		1✔
29	cmd := mybase.NewCommand("pull", summary, desc, PullHandler)	1✔
30	cmd.AddOption(mybase.BoolOption("include-auto-inc", 0, false, "Include starting auto-inc values in new table files, and update in existing files"))	1✔
31	cmd.AddOption(mybase.BoolOption("format", 0, true, "Reformat SQL statements to match canonical SHOW CREATE"))	1✔
32	cmd.AddOption(mybase.BoolOption("normalize", 0, true, "(deprecated alias for format)").Hidden().MarkDeprecated("This option will be removed in Skeema v2. Use the equivalent --format option instead, which currently defaults to true, but will default to false in Skeema v2."))	1✔
33	cmd.AddOption(mybase.BoolOption("new-schemas", 0, true, "Detect any new schemas and populate new dirs for them"))	1✔
34	cmd.AddOption(mybase.BoolOption("update-partitioning", 0, false, "Update PARTITION BY clauses in existing table files"))	1✔
35	cmd.AddOption(mybase.BoolOption("strip-partitioning", 0, false, "Omit PARTITION BY clause when writing partitioned tables to filesystem"))	1✔
36	workspace.AddCommandOptions(cmd)	1✔
37	cmd.AddArg("environment", "production", false)	1✔
38	CommandSuite.AddSubCommand(cmd)	1✔
39	}	1✔
40
41	// PullHandler is the handler method for `skeema pull`
42	func PullHandler(cfg *mybase.Config) error {	1✔
43	dir, err := fs.ParseDir(".", cfg)	1✔
44	if err != nil {	2✔
45	return err	1✔
46	}	1✔
47	if err := dir.CheckGenerator(generatorString()); err != nil {	2✔
48	return err	1✔
49	}	1✔
50	if !dir.Config.Supplied("format") && !dir.Config.Supplied("normalize") {	2✔
51	log.Debug("Upgrade notice: the --format option, which currently defaults to true in Skeema v1, will change to default to false in Skeema v2. For more information, visit https://www.skeema.io/v2-changes")	1✔
52	}	1✔
53
54	// pullWalker returns the "worst" (highest) exit code it encounters. We care
55	// about the exit code, but not the error message, since any error will already
56	// have been logged. (Multiple errors may have been encountered along the way,
57	// and it's simpler to log them when they occur, rather than needlessly
58	// collecting them.)
59	err = pullWalker(dir, 5)	1✔
60	return NewExitValue(ExitCode(err), "")	1✔
61	}
62
63	func pullWalker(dir *fs.Dir, maxDepth int) error {	1✔
64	if dir.ParseError != nil {	2✔
65	log.Warnf("Skipping %s: %s", dir, dir.ParseError)	1✔
66	return NewExitValue(CodeBadConfig, "")	1✔
67	}	1✔
68
69	var instance *tengo.Instance	1✔
70	var err error	1✔
71	if dir.Config.Changed("host") {	2✔
72	instance, err = dir.FirstInstance()	1✔
73	if err != nil {	2✔
74	log.Warnf("Skipping %s: %s", dir, err)	1✔
75	return NewExitValue(CodePartialError, "")	1✔
76	}	1✔
77	}
78
79	// dir defines a schema in .skeema, and/or has *.sql files
80	if dir.HasSchema() {	2✔
81	// If we cannot pull due to lack of an instance, make it clear to the user	1✔
82	if instance == nil {	2✔
83	log.Errorf("Skipping %s: No host defined for environment %q", dir, dir.Config.Get("environment"))	1✔
84	if dir.Config.Changed("host-wrapper") {	1✔
NEW 85	log.Error("The host-wrapper option controls which hosts this directory maps to, but the executed command did not output any hostnames.")	×
86	} else if rawHost := dir.Config.GetRaw("host"); rawHost != "" && rawHost[0] == '`' {	1✔
NEW 87	log.Error("The host option is set to a backtick-wrapped external command, but that command did not output any hostnames.")	×
88	} else {	1✔
89	log.Errorf("This command requires a hostname to pull from, which can be specified using the host option in a [%s] section of the .skeema file in this directory, or in a parent directory.", dir.Config.Get("environment"))	1✔
90	}	1✔
91	return NewExitValue(CodeBadConfig, "")	1✔
92	}
93
94	// Otherwise, we're in a "flat" dir that defines both host and schema: update
95	// the flavor if needed and process the pull operation on *.sql files, but no
96	// need to look for new schemas with this layout
97	updateFlavor(dir, instance)	1✔
98	updateGenerator(dir)	1✔
99	_, err := pullSchemaDir(dir, instance) // already logs err (if non-nil)	1✔
100	return err	1✔
101	}
102
103	subdirs, err := dir.Subdirs()	1✔
104	if err != nil {	1✔
105	log.Errorf("Cannot list subdirs of %s: %s", dir, err)	×
106	return NewExitValue(CodePartialError, "")	×
107	} else if len(subdirs) > 0 && maxDepth <= 0 {	1✔
108	log.Errorf("Not walking subdirs of %s: max depth reached", dir)	×
109	return NewExitValue(CodePartialError, "")	×
110	}	×
111
112	allSchemaNames := []string{}	1✔
113	for _, sub := range subdirs {	2✔
114	// If dir does not define host, simply recurse into subdirs.	1✔
115	if instance == nil {	2✔
116	subErr := pullWalker(sub, maxDepth-1)	1✔
117	err = HighestExitCode(err, subErr)	1✔
118	continue	1✔
119	}
120
121	// Otherwise, dir defines host but not schema. Treat subdirs as schema dirs,
122	// and use the combined list of handled schemas to figure out whether any
123	// new schema dirs need to be created (if requested).
124	subSchemaNames, subErr := pullSchemaDir(sub, instance) // already logs subErr (if non-nil)	1✔
125	err = HighestExitCode(err, subErr)	1✔
126	allSchemaNames = append(allSchemaNames, subSchemaNames...)	1✔
127	}
128
129	if instance != nil {	2✔
130	updateFlavor(dir, instance)	1✔
131	updateGenerator(dir)	1✔
132	if dir.Config.GetBool("new-schemas") && err == nil {	2✔
133	if err = findNewSchemas(dir, instance, allSchemaNames); err != nil {	1✔
134	log.Warnf("Unable to populate new schemas from %s: %s", dir, err)	×
135	return NewExitValue(CodePartialError, "")	×
136	}	×
137	}
138	}
139	return err	1✔
140	}
141
142	// pullSchemaDir updates all logical schemas in dir to reflect the actual
143	// definitions found in instance. A slice of handled schema names is returned,
144	// along with any error encountered.
145	func pullSchemaDir(dir fs.Dir, instance tengo.Instance) (schemaNames []string, err error) {	1✔
146	if dir.ParseError != nil {	2✔
147	log.Warnf("Skipping %s: %s", dir, dir.ParseError)	1✔
148	return nil, NewExitValue(CodePartialError, "")	1✔
149	}	1✔
150	if len(dir.LogicalSchemas) > 0 {	2✔
151	// TODO: support multiple logical schemas per dir	1✔
152	logicalSchema := dir.LogicalSchemas[0]	1✔
153	schemaNames, err = pullLogicalSchema(dir, instance, logicalSchema)	1✔
154	if err != nil {	2✔
155	log.Errorf("Skipping %s: %s\n", dir, err)	1✔
156	}	1✔
157	}
158	return	1✔
159	}
160
161	// pullLogicalSchema performs appropriate pull logic on a dir that maps to one or
162	// more schemas. A slice of handled schema names is returned, along with any
163	// error encountered.
164	func pullLogicalSchema(dir fs.Dir, instance tengo.Instance, logicalSchema *fs.LogicalSchema) (schemaNames []string, err error) {	1✔
165	// With non-zero lower_case_table_names, force names to lowercase as needed in	1✔
166	// logicalSchema, so that statements can be correctly linked to objects	1✔
167	if lctn := instance.NameCaseMode(); lctn > tengo.NameCaseAsIs {	2✔
168	if err := logicalSchema.LowerCaseNames(lctn); err != nil {	2✔
169	return nil, err	1✔
170	}	1✔
171	}
172
173	if logicalSchema.Name != "" {	1✔
174	schemaNames = []string{logicalSchema.Name}	×
175	} else if schemaNames, err = dir.SchemaNames(instance); err != nil {	2✔
176	return nil, fmt.Errorf("unable to fetch schema names mapped by this dir: %w", err)	1✔
177	}	1✔
178	if len(schemaNames) == 0 {	2✔
179	log.Warnf("Ignoring directory %s -- did not map to any schema names for environment %q\n", dir, dir.Config.Get("environment"))	1✔
180	return	1✔
181	}	1✔
182	instSchema, err := instance.Schema(schemaNames[0])	1✔
183	if err == sql.ErrNoRows {	2✔
184	log.Infof("Deleted directory %s -- schema %s no longer exists\n", dir, schemaNames[0])	1✔
185	return nil, dir.Delete()	1✔
186	} else if err != nil {	2✔
187	return nil, fmt.Errorf("Unable to fetch schema %s from %s: %s", schemaNames[0], instance, err)	×
188	}	×
189	instSchema.StripMatches(dir.IgnorePatterns)	1✔
190		1✔
191	log.Infof("Updating %s to reflect %s %s", dir, instance, instSchema.Name)	1✔
192		1✔
193	// Handle changes in schema's default character set and/or collation by	1✔
194	// persisting changes to the dir's option file.	1✔
195	if err := updateCharSetCollation(dir, instSchema); err != nil {	1✔
196	return nil, err	×
197	}	×
198
199	dumpOpts := dumper.Options{	1✔
200	IncludeAutoInc: dir.Config.GetBool("include-auto-inc"),	1✔
201	}	1✔
202	if !dir.Config.GetBool("update-partitioning") {	2✔
203	if dir.Config.GetBool("strip-partitioning") {	2✔
204	// Undocumented due to potential confusion, but supported just like in init	1✔
205	dumpOpts.Partitioning = tengo.PartitioningRemove	1✔
206	} else {	2✔
207	// Without --update-partitioning, retain whatever partitioning clause (or	1✔
208	// lack of clause) was already present in the *.sql files for existing tables.	1✔
209	dumpOpts.Partitioning = tengo.PartitioningKeep	1✔
210	}	1✔
211	}
212
213	// When --skip-format is in use, we only want to update objects that have
214	// actual functional modifications, NOT just cosmetic/formatting differences.
215	// To make this distinction, we need to actually execute the *.sql files in a
216	// Workspace and run a diff against it.
217	if !dir.Config.GetBool("format") \|\| !dir.Config.GetBool("normalize") {	2✔
218	mods := statementModifiersForPull(dir.Config, instance)	1✔
219	opts, err := workspace.OptionsForDir(dir, instance)	1✔
220	if err != nil {	2✔
221	return nil, WrapExitCode(CodeBadConfig, err)	1✔
222	}	1✔
223	inDiff, err := objectsInDiff(logicalSchema, instSchema, opts, mods)	1✔
224	if err != nil {	1✔
225	return nil, err	×
226	}	×
227	dumpOpts.OnlyKeys(inDiff)	1✔
228	}
229
230	_, err = dumper.DumpSchema(instSchema, dir, dumpOpts)	1✔
231	if err == nil {	2✔
232	os.Stderr.WriteString("\n")	1✔
233	}	1✔
234	return	1✔
235	}
236
237	func statementModifiersForPull(config mybase.Config, instance tengo.Instance) tengo.StatementModifiers {	1✔
238	// We're permissive of unsafe operations here since we don't ever actually	1✔
239	// execute the generated statement! We just examine its type.	1✔
240	mods := tengo.StatementModifiers{	1✔
241	AllowUnsafe: true,	1✔
242	}	1✔
243	// pull command updates next auto-increment value for existing table always	1✔
244	// if requested, or only if previously present in file otherwise	1✔
245	if config.GetBool("include-auto-inc") {	1✔
246	mods.NextAutoInc = tengo.NextAutoIncAlways	×
247	} else {	1✔
248	mods.NextAutoInc = tengo.NextAutoIncIfAlready	1✔
249	}	1✔
250	instFlavor, confFlavor := instance.Flavor(), tengo.ParseFlavor(config.Get("flavor"))	1✔
251	if !instFlavor.Known() && confFlavor.Known() {	1✔
252	mods.Flavor = confFlavor	×
253	} else {	1✔
254	mods.Flavor = instFlavor	1✔
255	}	1✔
256	// Unless user specifically wants to update partitioning clauses, apply a
257	// statement modifier to make some partitioning-related AlterClause types
258	// return an empty statement, to exclude them from being rewritten if their
259	// only differences are partitioning-related.
260	if !config.GetBool("update-partitioning") {	2✔
261	mods.Partitioning = tengo.PartitioningKeep	1✔
262	}	1✔
263	return mods	1✔
264	}
265
266	// objectsInDiff returns a map whose keys are tengo.ObjectKeys of objects that
267	// have modifications in instSchema that aren't reflected in their filesystem
268	// representation yet. This also includes objects whose filesystem Statement has
269	// a SQL syntax error. The return value does not include tables whose
270	// differences are cosmetic / formatting-related, or are otherwise ignored by
271	// mods.
272	func objectsInDiff(logicalSchema fs.LogicalSchema, instSchema tengo.Schema, opts workspace.Options, mods tengo.StatementModifiers) ([]tengo.ObjectKey, error) {	1✔
273	wsSchema, err := workspace.ExecLogicalSchema(logicalSchema, opts)	1✔
274	if err != nil {	1✔
275	return nil, fmt.Errorf("Error introspecting filesystem version of schema %s: %s", instSchema.Name, err)	×
276	}	×
277	log.Debugf("Workspace performance using %s:\n%s", wsSchema.Info, wsSchema.Timers)	1✔
278		1✔
279	// Run a diff, and create a map to track objects in the diff	1✔
280	diff := tengo.NewSchemaDiff(wsSchema.Schema, instSchema)	1✔
281	inDiff := make([]tengo.ObjectKey, 0)	1✔
282	for _, od := range diff.ObjectDiffs() {	2✔
283	odStatement, odStatementErr := od.Statement(mods)	1✔
284	key := od.ObjectKey()	1✔
285	// Errors are fatal, except for UnsupportedDiffError which we can safely	1✔
286	// ignore (since pull doesn't actually run ALTERs; it just needs to know	1✔
287	// what was altered)	1✔
288	if odStatementErr != nil && !tengo.IsUnsupportedDiff(odStatementErr) {	1✔
289	return nil, odStatementErr	×
290	}	×
291	// mods may cause the diff to be a no-op; only include it in result if this
292	// isn't the case
293	if odStatement != "" {	2✔
294	inDiff = append(inDiff, key)	1✔
295	}	1✔
296	}
297
298	// Treat objects with syntax errors as modified, since it isn't possible for
299	// the filesystem definition to match the live definition in this case.
300	inDiff = append(inDiff, wsSchema.FailedKeys()...)	1✔
301		1✔
302	return inDiff, nil	1✔
303	}
304
305	// updateFlavor updates the dir's .skeema option file if the instance's current
306	// flavor does not match what's in the file. However, it leaves the value in the
307	// file alone if it's specified and we're unable to detect the instance's
308	// vendor, as this gives operators the ability to manually override an
309	// undetectable flavor.
310	func updateFlavor(dir fs.Dir, instance tengo.Instance) {	1✔
311	instFlavor := instance.Flavor()	1✔
312	if !instFlavor.Known() \|\| instFlavor.Family().String() == dir.Config.Get("flavor") {	2✔
313	return	1✔
314	}	1✔
315	dir.OptionFile.SetOptionValue(dir.Config.Get("environment"), "flavor", instFlavor.Family().String())	1✔
316	if err := dir.OptionFile.Write(true); err != nil {	1✔
317	log.Warnf("Unable to update flavor in %s: %s", dir.OptionFile.Path(), err)	×
318	} else {	1✔
319	log.Infof("Wrote %s -- updated flavor to %s", dir.OptionFile.Path(), instFlavor.Family().String())	1✔
320	}	1✔
321	}
322
323	func updateGenerator(dir *fs.Dir) {	1✔
324	currentGenerator := generatorString()	1✔
325	if dir.Config.Get("generator") == currentGenerator {	2✔
326	return	1✔
327	}	1✔
328	dir.OptionFile.SetOptionValue("", "generator", currentGenerator)	1✔
329	// ignoring errors; failure not terribly important	1✔
330	if err := dir.OptionFile.Write(true); err == nil {	2✔
331	log.Infof("Wrote %s -- updated generator to %s", dir.OptionFile.Path(), currentGenerator)	1✔
332	}	1✔
333	}
334
335	// updateCharSetCollation updates the dir's .skeema option file if the schema's
336	// current default charset or collation does not match what's in the file.
337	func updateCharSetCollation(dir fs.Dir, instSchema tengo.Schema) error {	1✔
338	if dir.Config.Get("default-character-set") != instSchema.CharSet \|\| dir.Config.Get("default-collation") != instSchema.Collation {	2✔
339	dir.OptionFile.SetOptionValue("", "default-character-set", instSchema.CharSet)	1✔
340	dir.OptionFile.SetOptionValue("", "default-collation", instSchema.Collation)	1✔
341	if err := dir.OptionFile.Write(true); err != nil {	1✔
342	return fmt.Errorf("Unable to update character set and collation for %s: %s", dir.OptionFile.Path(), err)	×
343	}	×
344	log.Infof("Wrote %s -- updated schema-level default-character-set and default-collation", dir.OptionFile.Path())	1✔
345	}
346	return nil	1✔
347	}
348
349	func findNewSchemas(dir fs.Dir, instance tengo.Instance, seenNames []string) error {	1✔
350	subdirHasSchema := make(map[string]bool)	1✔
351	for _, name := range seenNames {	2✔
352	subdirHasSchema[name] = true	1✔
353	}	1✔
354
355	schemaNames, err := instance.SchemaNames()	1✔
356	if err != nil {	1✔
357	return err	×
358	}	×
359	alreadyWarned := dir.Config.Supplied("new-schemas") // no need to warn if option set explicitly	1✔
360	for _, name := range schemaNames {	2✔
361	// If no existing subdir maps to the schema, we need to create and populate new dir	1✔
362	if !subdirHasSchema[name] {	2✔
363	if !alreadyWarned {	2✔
364	alreadyWarned = true	1✔
365	log.Debug("Upgrade notice: the --new-schemas option, which currently defaults to true in Skeema v1, will change to default to false in Skeema v2. For more information, visit https://www.skeema.io/v2-changes")	1✔
366	}	1✔
367	s, err := instance.Schema(name)	1✔
368	if err != nil {	1✔
369	return err	×
370	}	×
371	s.StripMatches(dir.IgnorePatterns)	1✔
372	// use same logic from init command	1✔
373	if err := PopulateSchemaDir(s, dir, true); err != nil {	1✔
374	return err	×
375	}	×
376	}
377	}
378
379	return nil	1✔
380	}

skeema / skeema / 25829447167

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous