21765311924

Committed 06 Feb 2026 08:44PM UTC coverage: 80.83% (-0.09%) from 80.924%

Build # 21765311924

Build Type

Pull #158

github

Committed by

web-flow

Commit Message

Merge 2c4faaa9f into 94c30c8e7

Pull Request Pull Request #158: feat(deployment-policy): add batch state reset with auto-reset, cli, and config

Coverage Stats

241 of 313 new or added lines in 10 files covered. (77.0%)

25 existing lines in 5 files now uncovered.

6818 of 8435 relevant lines covered (80.83%)

4.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

77.12

/operator/cmd/cli/app/lifecycle.go

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package app

import (
        "bufio"
        "fmt"
        "strings"

        "github.com/spf13/cobra"

        "github.com/NVIDIA/skyhook/operator/internal/cli/client"
        cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
        "github.com/NVIDIA/skyhook/operator/internal/cli/utils"
)

// lifecycleConfig defines the configuration for a lifecycle command
type lifecycleConfig struct {
        use          string
        short        string
        long         string
        example      string
        annotation   string
        action       string // "set" or "remove"
        verb         string // past tense for output message (e.g., "paused", "resumed")
        confirmVerb  string // verb for confirmation prompt (e.g., "pause", "disable")
        needsConfirm bool
}

// lifecycleOptions holds the options for lifecycle commands that need confirmation
type lifecycleOptions struct {
        confirm bool
}

// newLifecycleCmd creates a lifecycle command based on the provided configuration
func newLifecycleCmd(ctx *cliContext.CLIContext, cfg lifecycleConfig) *cobra.Command {
        opts := &lifecycleOptions{}

        cmd := &cobra.Command{
                Use:     cfg.use,
                Short:   cfg.short,
                Long:    cfg.long,
                Example: cfg.example,
                Args:    cobra.ExactArgs(1),
                RunE: func(cmd *cobra.Command, args []string) error {
                        skyhookName := args[0]

                        if cfg.needsConfirm && !opts.confirm {
                                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "This will %s Skyhook %q. Continue? [y/N]: ",
                                        cfg.confirmVerb, skyhookName)
                                reader := bufio.NewReader(cmd.InOrStdin())
                                response, err := reader.ReadString('\n')
                                if err != nil {
                                        _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Aborted.")
                                        return nil
                                }
                                response = strings.TrimSpace(response)
                                if response != "y" && response != "Y" {
                                        _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Aborted.")
                                        return nil
                                }
                        }

                        // Check dry-run before making changes
                        if ctx.GlobalFlags.DryRun {
                                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "[dry-run] Would %s Skyhook %q\n", cfg.confirmVerb, skyhookName)
                                return nil
                        }

                        clientFactory := client.NewFactory(ctx.GlobalFlags.ConfigFlags)
                        kubeClient, err := clientFactory.Client()
                        if err != nil {
                                return fmt.Errorf("initializing kubernetes client: %w", err)
                        }

                        // Fetch the Skyhook to check operator version from its annotation
                        skyhook, err := utils.GetSkyhook(cmd.Context(), kubeClient.Dynamic(), skyhookName)
                        if err != nil {
                                return err
                        }

                        // Check operator version compatibility using Skyhook's version annotation
                        // If annotation is missing or invalid (e.g., "dev"), fall back to deployment version
                        opVersion := utils.GetSkyhookVersion(skyhook)
                        if opVersion == "" || !utils.IsValidVersion(opVersion) {
                                // Try to get version from deployment instead
                                deployVersion, err := utils.DiscoverOperatorVersion(cmd.Context(), kubeClient.Kubernetes(), ctx.GlobalFlags.Namespace())
                                if err == nil && utils.IsValidVersion(deployVersion) {
                                        opVersion = deployVersion
                                } else {
                                        _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: unable to determine operator version; "+
                                                "cannot verify compatibility (requires %s+)\n", utils.MinAnnotationSupportVersion)
                                }
                        }

                        if utils.IsValidVersion(opVersion) && utils.CompareVersions(opVersion, utils.MinAnnotationSupportVersion) < 0 {
                                // Operator is older than v0.8.0 - annotations won't work
                                if cfg.annotation == utils.PauseAnnotation {
                                        // pause/resume - feature exists but uses spec field instead of annotation
                                        specValue := "true"
                                        if cfg.action == "remove" {
                                                specValue = "false"
                                        }
                                        return fmt.Errorf("operator version %s uses spec.pause instead of annotations; "+
                                                "use 'kubectl edit skyhook %s' and set spec.pause: %s", opVersion, skyhookName, specValue)
                                }
                                // disable/enable - feature doesn't exist at all in older versions
                                return fmt.Errorf("operator version %s does not support %s; this feature was added in %s",
                                        opVersion, cfg.confirmVerb, utils.MinAnnotationSupportVersion)
                        }

                        if cfg.action == "set" {
                                err = utils.SetSkyhookAnnotation(cmd.Context(), kubeClient.Dynamic(), skyhookName, cfg.annotation, "true")
                        } else {
                                err = utils.RemoveSkyhookAnnotation(cmd.Context(), kubeClient.Dynamic(), skyhookName, cfg.annotation)
                        }
                        if err != nil {
                                return err
                        }

                        _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Skyhook %q %s\n", skyhookName, cfg.verb)
                        return nil
                },
        }

        if cfg.needsConfirm {
                cmd.Flags().BoolVarP(&opts.confirm, "confirm", "y", false, "Skip confirmation prompt")
        }

        return cmd
}

// NewPauseCmd creates the pause command
func NewPauseCmd(ctx *cliContext.CLIContext) *cobra.Command {
        return newLifecycleCmd(ctx, lifecycleConfig{
                use:   "pause <skyhook-name>",
                short: "Pause a Skyhook from processing",
                long: `Pause a Skyhook by setting the pause annotation.

When a Skyhook is paused, the operator will stop processing new nodes
but will not interrupt any currently running operations.`,
                example: `  # Pause a Skyhook
  kubectl skyhook pause gpu-init

  # Pause without confirmation
  kubectl skyhook pause gpu-init --confirm`,
                annotation:   utils.PauseAnnotation,
                action:       "set",
                verb:         "paused",
                confirmVerb:  "pause",
                needsConfirm: true,
        })
}

// NewResumeCmd creates the resume command
func NewResumeCmd(ctx *cliContext.CLIContext) *cobra.Command {
        return newLifecycleCmd(ctx, lifecycleConfig{
                use:   "resume <skyhook-name>",
                short: "Resume a paused Skyhook",
                long: `Resume a paused Skyhook by removing the pause annotation.

The operator will resume processing nodes after this command.`,
                example: `  # Resume a paused Skyhook
  kubectl skyhook resume gpu-init

  # Resume without confirmation
  kubectl skyhook resume gpu-init --confirm`,
                annotation:   utils.PauseAnnotation,
                action:       "remove",
                verb:         "resumed",
                confirmVerb:  "resume",
                needsConfirm: true,
        })
}

// NewDisableCmd creates the disable command
func NewDisableCmd(ctx *cliContext.CLIContext) *cobra.Command {
        return newLifecycleCmd(ctx, lifecycleConfig{
                use:   "disable <skyhook-name>",
                short: "Disable a Skyhook completely",
                long: `Disable a Skyhook by setting the disable annotation.

When a Skyhook is disabled, the operator will completely stop processing
and the Skyhook will be effectively inactive.`,
                example: `  # Disable a Skyhook
  kubectl skyhook disable gpu-init

  # Disable without confirmation
  kubectl skyhook disable gpu-init --confirm`,
                annotation:   utils.DisableAnnotation,
                action:       "set",
                verb:         "disabled",
                confirmVerb:  "disable",
                needsConfirm: true,
        })
}

// NewEnableCmd creates the enable command
func NewEnableCmd(ctx *cliContext.CLIContext) *cobra.Command {
        return newLifecycleCmd(ctx, lifecycleConfig{
                use:   "enable <skyhook-name>",
                short: "Enable a disabled Skyhook",
                long: `Enable a disabled Skyhook by removing the disable annotation.

The operator will resume normal processing after this command.`,
                example: `  # Enable a disabled Skyhook
  kubectl skyhook enable gpu-init

  # Enable without confirmation
  kubectl skyhook enable gpu-init --confirm`,
                annotation:   utils.DisableAnnotation,
                action:       "remove",
                verb:         "enabled",
                confirmVerb:  "enable",
                needsConfirm: true,
        })
}

1	/*
2	* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3	* SPDX-License-Identifier: Apache-2.0
4	*
5	*
6	* Licensed under the Apache License, Version 2.0 (the "License");
7	* you may not use this file except in compliance with the License.
8	* You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing, software
13	* distributed under the License is distributed on an "AS IS" BASIS,
14	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	* See the License for the specific language governing permissions and
16	* limitations under the License.
17	*/
18
19	package app
20
21	import (
22	"bufio"
23	"fmt"
24	"strings"
25
26	"github.com/spf13/cobra"
27
28	"github.com/NVIDIA/skyhook/operator/internal/cli/client"
29	cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
30	"github.com/NVIDIA/skyhook/operator/internal/cli/utils"
31	)
32
33	// lifecycleConfig defines the configuration for a lifecycle command
34	type lifecycleConfig struct {
35	use string
36	short string
37	long string
38	example string
39	annotation string
40	action string // "set" or "remove"
41	verb string // past tense for output message (e.g., "paused", "resumed")
42	confirmVerb string // verb for confirmation prompt (e.g., "pause", "disable")
43	needsConfirm bool
44	}
45
46	// lifecycleOptions holds the options for lifecycle commands that need confirmation
47	type lifecycleOptions struct {
48	confirm bool
49	}
50
51	// newLifecycleCmd creates a lifecycle command based on the provided configuration
52	func newLifecycleCmd(ctx cliContext.CLIContext, cfg lifecycleConfig) cobra.Command {	2✔
53	opts := &lifecycleOptions{}	2✔
54		2✔
55	cmd := &cobra.Command{	2✔
56	Use: cfg.use,	2✔
57	Short: cfg.short,	2✔
58	Long: cfg.long,	2✔
59	Example: cfg.example,	2✔
60	Args: cobra.ExactArgs(1),	2✔
61	RunE: func(cmd *cobra.Command, args []string) error {	3✔
62	skyhookName := args[0]	1✔
63		1✔
64	if cfg.needsConfirm && !opts.confirm {	2✔
65	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "This will %s Skyhook %q. Continue? [y/N]: ",	1✔
66	cfg.confirmVerb, skyhookName)	1✔
67	reader := bufio.NewReader(cmd.InOrStdin())	1✔
68	response, err := reader.ReadString('\n')	1✔
69	if err != nil {	2✔
70	_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Aborted.")	1✔
71	return nil	1✔
72	}	1✔
73	response = strings.TrimSpace(response)	×
74	if response != "y" && response != "Y" {	×
75	_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Aborted.")	×
76	return nil	×
77	}	×
78	}
79
80	// Check dry-run before making changes
81	if ctx.GlobalFlags.DryRun {	1✔
82	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "[dry-run] Would %s Skyhook %q\n", cfg.confirmVerb, skyhookName)	×
83	return nil	×
84	}	×
85
86	clientFactory := client.NewFactory(ctx.GlobalFlags.ConfigFlags)	1✔
87	kubeClient, err := clientFactory.Client()	1✔
88	if err != nil {	1✔
89	return fmt.Errorf("initializing kubernetes client: %w", err)	×
90	}	×
91
92	// Fetch the Skyhook to check operator version from its annotation
93	skyhook, err := utils.GetSkyhook(cmd.Context(), kubeClient.Dynamic(), skyhookName)	1✔
94	if err != nil {	1✔
95	return err	×
96	}	×
97
98	// Check operator version compatibility using Skyhook's version annotation
99	// If annotation is missing or invalid (e.g., "dev"), fall back to deployment version
100	opVersion := utils.GetSkyhookVersion(skyhook)	1✔
101	if opVersion == "" \|\| !utils.IsValidVersion(opVersion) {	1✔
UNCOV 102	// Try to get version from deployment instead	×
UNCOV 103	deployVersion, err := utils.DiscoverOperatorVersion(cmd.Context(), kubeClient.Kubernetes(), ctx.GlobalFlags.Namespace())	×
UNCOV 104	if err == nil && utils.IsValidVersion(deployVersion) {	×
105	opVersion = deployVersion	×
UNCOV 106	} else {	×
UNCOV 107	_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: unable to determine operator version; "+	×
UNCOV 108	"cannot verify compatibility (requires %s+)\n", utils.MinAnnotationSupportVersion)	×
UNCOV 109	}	×
110	}
111
112	if utils.IsValidVersion(opVersion) && utils.CompareVersions(opVersion, utils.MinAnnotationSupportVersion) < 0 {	1✔
113	// Operator is older than v0.8.0 - annotations won't work	×
114	if cfg.annotation == utils.PauseAnnotation {	×
115	// pause/resume - feature exists but uses spec field instead of annotation	×
116	specValue := "true"	×
117	if cfg.action == "remove" {	×
118	specValue = "false"	×
119	}	×
120	return fmt.Errorf("operator version %s uses spec.pause instead of annotations; "+	×
121	"use 'kubectl edit skyhook %s' and set spec.pause: %s", opVersion, skyhookName, specValue)	×
122	}
123	// disable/enable - feature doesn't exist at all in older versions
124	return fmt.Errorf("operator version %s does not support %s; this feature was added in %s",	×
125	opVersion, cfg.confirmVerb, utils.MinAnnotationSupportVersion)	×
126	}
127
128	if cfg.action == "set" {	2✔
129	err = utils.SetSkyhookAnnotation(cmd.Context(), kubeClient.Dynamic(), skyhookName, cfg.annotation, "true")	1✔
130	} else {	1✔
131	err = utils.RemoveSkyhookAnnotation(cmd.Context(), kubeClient.Dynamic(), skyhookName, cfg.annotation)	×
132	}	×
133	if err != nil {	1✔
134	return err	×
135	}	×
136
137	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Skyhook %q %s\n", skyhookName, cfg.verb)	1✔
138	return nil	1✔
139	},
140	}
141
142	if cfg.needsConfirm {	4✔
143	cmd.Flags().BoolVarP(&opts.confirm, "confirm", "y", false, "Skip confirmation prompt")	2✔
144	}	2✔
145
146	return cmd	2✔
147	}
148
149	// NewPauseCmd creates the pause command
150	func NewPauseCmd(ctx cliContext.CLIContext) cobra.Command {	2✔
151	return newLifecycleCmd(ctx, lifecycleConfig{	2✔
152	use: "pause <skyhook-name>",	2✔
153	short: "Pause a Skyhook from processing",	2✔
154	long: `Pause a Skyhook by setting the pause annotation.	2✔
155		2✔
156	When a Skyhook is paused, the operator will stop processing new nodes	2✔
157	but will not interrupt any currently running operations.`,	2✔
158	example: ` # Pause a Skyhook	2✔
159	kubectl skyhook pause gpu-init	2✔
160		2✔
161	# Pause without confirmation	2✔
162	kubectl skyhook pause gpu-init --confirm`,	2✔
163	annotation: utils.PauseAnnotation,	2✔
164	action: "set",	2✔
165	verb: "paused",	2✔
166	confirmVerb: "pause",	2✔
167	needsConfirm: true,	2✔
168	})	2✔
169	}	2✔
170
171	// NewResumeCmd creates the resume command
172	func NewResumeCmd(ctx cliContext.CLIContext) cobra.Command {	2✔
173	return newLifecycleCmd(ctx, lifecycleConfig{	2✔
174	use: "resume <skyhook-name>",	2✔
175	short: "Resume a paused Skyhook",	2✔
176	long: `Resume a paused Skyhook by removing the pause annotation.	2✔
177		2✔
178	The operator will resume processing nodes after this command.`,	2✔
179	example: ` # Resume a paused Skyhook	2✔
180	kubectl skyhook resume gpu-init	2✔
181		2✔
182	# Resume without confirmation	2✔
183	kubectl skyhook resume gpu-init --confirm`,	2✔
184	annotation: utils.PauseAnnotation,	2✔
185	action: "remove",	2✔
186	verb: "resumed",	2✔
187	confirmVerb: "resume",	2✔
188	needsConfirm: true,	2✔
189	})	2✔
190	}	2✔
191
192	// NewDisableCmd creates the disable command
193	func NewDisableCmd(ctx cliContext.CLIContext) cobra.Command {	2✔
194	return newLifecycleCmd(ctx, lifecycleConfig{	2✔
195	use: "disable <skyhook-name>",	2✔
196	short: "Disable a Skyhook completely",	2✔
197	long: `Disable a Skyhook by setting the disable annotation.	2✔
198		2✔
199	When a Skyhook is disabled, the operator will completely stop processing	2✔
200	and the Skyhook will be effectively inactive.`,	2✔
201	example: ` # Disable a Skyhook	2✔
202	kubectl skyhook disable gpu-init	2✔
203		2✔
204	# Disable without confirmation	2✔
205	kubectl skyhook disable gpu-init --confirm`,	2✔
206	annotation: utils.DisableAnnotation,	2✔
207	action: "set",	2✔
208	verb: "disabled",	2✔
209	confirmVerb: "disable",	2✔
210	needsConfirm: true,	2✔
211	})	2✔
212	}	2✔
213
214	// NewEnableCmd creates the enable command
215	func NewEnableCmd(ctx cliContext.CLIContext) cobra.Command {	2✔
216	return newLifecycleCmd(ctx, lifecycleConfig{	2✔
217	use: "enable <skyhook-name>",	2✔
218	short: "Enable a disabled Skyhook",	2✔
219	long: `Enable a disabled Skyhook by removing the disable annotation.	2✔
220		2✔
221	The operator will resume normal processing after this command.`,	2✔
222	example: ` # Enable a disabled Skyhook	2✔
223	kubectl skyhook enable gpu-init	2✔
224		2✔
225	# Enable without confirmation	2✔
226	kubectl skyhook enable gpu-init --confirm`,	2✔
227	annotation: utils.DisableAnnotation,	2✔
228	action: "remove",	2✔
229	verb: "enabled",	2✔
230	confirmVerb: "enable",	2✔
231	needsConfirm: true,	2✔
232	})	2✔
233	}	2✔

NVIDIA / skyhook / 21765311924

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous