20106791825

Committed 10 Dec 2025 04:59PM UTC coverage: 77.027% (+2.1%) from 74.963%

Build # 20106791825

Build Type

Pull #125

github

Committed by

t0mmylam

Commit Message

feat: Consolidate CLI e2e tests with proper assertions and CI integration

Pull Request Pull Request #125: feat: Create CLI e2e tests with assertions and CI integration

Run Details

1059 of 1277 new or added lines in 13 files covered. (82.93%)

1 existing line in 1 file now uncovered.

5700 of 7400 relevant lines covered (77.03%)

0.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.33

/operator/internal/cli/node/node_reset.go

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package node

import (
        "bufio"
        "context"
        "encoding/json"
        "fmt"
        "strings"

        "github.com/spf13/cobra"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "github.com/NVIDIA/skyhook/operator/api/v1alpha1"
        "github.com/NVIDIA/skyhook/operator/internal/cli/client"
        cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
        "github.com/NVIDIA/skyhook/operator/internal/cli/utils"
)

// nodeResetOptions holds the options for the node reset command
type nodeResetOptions struct {
        skyhookName string
        confirm     bool
}

// NewResetCmd creates the node reset command
func NewResetCmd(ctx *cliContext.CLIContext) *cobra.Command {
        opts := &nodeResetOptions{}

        cmd := &cobra.Command{
                Use:   "reset <node-name...>",
                Short: "Reset all package state on node(s) for a Skyhook",
                Long: `Reset all package state on node(s) for a specific Skyhook, forcing a complete re-run.

This command removes all Skyhook state from the specified node(s), causing
the operator to re-execute all packages from the beginning.

Unlike 'package rerun' which resets a single package, 'node reset' clears
ALL package state for a Skyhook on the specified node(s).

Node names can be exact matches or regex patterns.`,
                Example: `  # Reset all packages on worker-1 for gpu-init Skyhook
  kubectl skyhook node reset worker-1 --skyhook gpu-init --confirm

  # Reset multiple nodes
  kubectl skyhook node reset worker-1 worker-2 worker-3 --skyhook gpu-init --confirm

  # Reset all nodes matching a pattern
  kubectl skyhook node reset "gpu-node-.*" --skyhook gpu-init --confirm

  # Preview changes without applying (dry-run)
  kubectl skyhook node reset worker-1 --skyhook gpu-init --dry-run`,
                Args: cobra.MinimumNArgs(1),
                RunE: func(cmd *cobra.Command, args []string) error {
                        if opts.skyhookName == "" {
                                return fmt.Errorf("--skyhook flag is required")
                        }

                        clientFactory := client.NewFactory(ctx.GlobalFlags.ConfigFlags)
                        kubeClient, err := clientFactory.Client()
                        if err != nil {
                                return fmt.Errorf("initializing kubernetes client: %w", err)
                        }

                        return runNodeReset(cmd.Context(), cmd, kubeClient, args, opts, ctx)
                },
        }

        cmd.Flags().StringVar(&opts.skyhookName, "skyhook", "", "Name of the Skyhook CR (required)")
        cmd.Flags().BoolVarP(&opts.confirm, "confirm", "y", false, "Skip confirmation prompt")

        _ = cmd.MarkFlagRequired("skyhook")

        return cmd
}

func runNodeReset(ctx context.Context, cmd *cobra.Command, kubeClient *client.Client, nodePatterns []string, opts *nodeResetOptions, cliCtx *cliContext.CLIContext) error {
        // Get all nodes
        nodeList, err := kubeClient.Kubernetes().CoreV1().Nodes().List(ctx, metav1.ListOptions{})
        if err != nil {
                return fmt.Errorf("listing nodes: %w", err)
        }

        // Collect all node names for pattern matching
        allNodeNames := make([]string, 0, len(nodeList.Items))
        nodeMap := make(map[string]int) // node name -> index in nodeList.Items
        for i, node := range nodeList.Items {
                allNodeNames = append(allNodeNames, node.Name)
                nodeMap[node.Name] = i
        }

        // Match nodes
        matchedNodes, err := utils.MatchNodes(nodePatterns, allNodeNames)
        if err != nil {
                return fmt.Errorf("matching nodes: %w", err)
        }

        if len(matchedNodes) == 0 {
                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "No nodes matched the specified patterns\n")
                return nil
        }

        // Find nodes that have the specified Skyhook annotation
        annotationKey := nodeStateAnnotationPrefix + opts.skyhookName
        nodesToReset := make([]string, 0, len(matchedNodes))
        nodeStates := make(map[string]v1alpha1.NodeState)

        for _, nodeName := range matchedNodes {
                idx := nodeMap[nodeName]
                node := &nodeList.Items[idx]

                annotation, ok := node.Annotations[annotationKey]
                if !ok {
                        continue
                }

                var nodeState v1alpha1.NodeState
                if err := json.Unmarshal([]byte(annotation), &nodeState); err != nil {
                        continue
                }

                nodesToReset = append(nodesToReset, nodeName)
                nodeStates[nodeName] = nodeState
        }

        if len(nodesToReset) == 0 {
                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "No nodes have state for Skyhook %q\n", opts.skyhookName)
                return nil
        }

        // Print summary
        _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Skyhook: %s\n", opts.skyhookName)
        _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Nodes to reset (%d):\n", len(nodesToReset))
        for _, nodeName := range nodesToReset {
                nodeState := nodeStates[nodeName]
                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "  - %s (%d packages)\n", nodeName, len(nodeState))
        }

        // Dry run check
        if cliCtx.GlobalFlags.DryRun {
                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\n[dry-run] No changes applied\n")
                return nil
        }

        // Confirmation
        if !opts.confirm {
                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nThis will remove ALL package state for Skyhook %q on these nodes.\n", opts.skyhookName)
                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "All packages will re-run from the beginning.\n")
                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Continue? [y/N]: ")

                reader := bufio.NewReader(cmd.InOrStdin())
                response, err := reader.ReadString('\n')
                if err != nil {
                        return fmt.Errorf("reading confirmation: %w", err)
                }

                response = strings.ToLower(strings.TrimSpace(response))
                if response != "y" && response != "yes" {
                        _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Aborted\n")
                        return nil
                }
        }

        // Apply changes
        var updateErrors []string
        successCount := 0

        for _, nodeName := range nodesToReset {
                idx := nodeMap[nodeName]
                node := nodeList.Items[idx].DeepCopy()

                // Remove the Skyhook annotation
                delete(node.Annotations, annotationKey)

                _, err := kubeClient.Kubernetes().CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{})
                if err != nil {
                        updateErrors = append(updateErrors, fmt.Sprintf("%s: %v", nodeName, err))
                        continue
                }
                successCount++
        }

        // Print results
        if len(updateErrors) > 0 {
                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nErrors resetting some nodes:\n")
                for _, e := range updateErrors {
                        _, _ = fmt.Fprintf(cmd.OutOrStdout(), "  - %s\n", e)
                }
        }

        if successCount > 0 {
                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nSuccessfully reset %d node(s) for Skyhook %q\n", successCount, opts.skyhookName)
        }

        return nil
}

1	/*
2	* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3	* SPDX-License-Identifier: Apache-2.0
4	*
5	*
6	* Licensed under the Apache License, Version 2.0 (the "License");
7	* you may not use this file except in compliance with the License.
8	* You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing, software
13	* distributed under the License is distributed on an "AS IS" BASIS,
14	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	* See the License for the specific language governing permissions and
16	* limitations under the License.
17	*/
18
19	package node
20
21	import (
22	"bufio"
23	"context"
24	"encoding/json"
25	"fmt"
26	"strings"
27
28	"github.com/spf13/cobra"
29	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30
31	"github.com/NVIDIA/skyhook/operator/api/v1alpha1"
32	"github.com/NVIDIA/skyhook/operator/internal/cli/client"
33	cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
34	"github.com/NVIDIA/skyhook/operator/internal/cli/utils"
35	)
36
37	// nodeResetOptions holds the options for the node reset command
38	type nodeResetOptions struct {
39	skyhookName string
40	confirm bool
41	}
42
43	// NewResetCmd creates the node reset command
44	func NewResetCmd(ctx cliContext.CLIContext) cobra.Command {	1✔
45	opts := &nodeResetOptions{}	1✔
46		1✔
47	cmd := &cobra.Command{	1✔
48	Use: "reset <node-name...>",	1✔
49	Short: "Reset all package state on node(s) for a Skyhook",	1✔
50	Long: `Reset all package state on node(s) for a specific Skyhook, forcing a complete re-run.	1✔
51		1✔
52	This command removes all Skyhook state from the specified node(s), causing	1✔
53	the operator to re-execute all packages from the beginning.	1✔
54		1✔
55	Unlike 'package rerun' which resets a single package, 'node reset' clears	1✔
56	ALL package state for a Skyhook on the specified node(s).	1✔
57		1✔
58	Node names can be exact matches or regex patterns.`,	1✔
59	Example: ` # Reset all packages on worker-1 for gpu-init Skyhook	1✔
60	kubectl skyhook node reset worker-1 --skyhook gpu-init --confirm	1✔
61		1✔
62	# Reset multiple nodes	1✔
63	kubectl skyhook node reset worker-1 worker-2 worker-3 --skyhook gpu-init --confirm	1✔
64		1✔
65	# Reset all nodes matching a pattern	1✔
66	kubectl skyhook node reset "gpu-node-.*" --skyhook gpu-init --confirm	1✔
67		1✔
68	# Preview changes without applying (dry-run)	1✔
69	kubectl skyhook node reset worker-1 --skyhook gpu-init --dry-run`,	1✔
70	Args: cobra.MinimumNArgs(1),	1✔
71	RunE: func(cmd *cobra.Command, args []string) error {	1✔
NEW 72	if opts.skyhookName == "" {	×
NEW 73	return fmt.Errorf("--skyhook flag is required")	×
NEW 74	}	×
75
NEW 76	clientFactory := client.NewFactory(ctx.GlobalFlags.ConfigFlags)	×
NEW 77	kubeClient, err := clientFactory.Client()	×
NEW 78	if err != nil {	×
NEW 79	return fmt.Errorf("initializing kubernetes client: %w", err)	×
NEW 80	}	×
81
NEW 82	return runNodeReset(cmd.Context(), cmd, kubeClient, args, opts, ctx)	×
83	},
84	}
85
86	cmd.Flags().StringVar(&opts.skyhookName, "skyhook", "", "Name of the Skyhook CR (required)")	1✔
87	cmd.Flags().BoolVarP(&opts.confirm, "confirm", "y", false, "Skip confirmation prompt")	1✔
88		1✔
89	_ = cmd.MarkFlagRequired("skyhook")	1✔
90		1✔
91	return cmd	1✔
92	}
93
94	func runNodeReset(ctx context.Context, cmd cobra.Command, kubeClient client.Client, nodePatterns []string, opts nodeResetOptions, cliCtx cliContext.CLIContext) error {	1✔
95	// Get all nodes	1✔
96	nodeList, err := kubeClient.Kubernetes().CoreV1().Nodes().List(ctx, metav1.ListOptions{})	1✔
97	if err != nil {	1✔
NEW 98	return fmt.Errorf("listing nodes: %w", err)	×
NEW 99	}	×
100
101	// Collect all node names for pattern matching
102	allNodeNames := make([]string, 0, len(nodeList.Items))	1✔
103	nodeMap := make(map[string]int) // node name -> index in nodeList.Items	1✔
104	for i, node := range nodeList.Items {	2✔
105	allNodeNames = append(allNodeNames, node.Name)	1✔
106	nodeMap[node.Name] = i	1✔
107	}	1✔
108
109	// Match nodes
110	matchedNodes, err := utils.MatchNodes(nodePatterns, allNodeNames)	1✔
111	if err != nil {	1✔
NEW 112	return fmt.Errorf("matching nodes: %w", err)	×
NEW 113	}	×
114
115	if len(matchedNodes) == 0 {	2✔
116	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "No nodes matched the specified patterns\n")	1✔
117	return nil	1✔
118	}	1✔
119
120	// Find nodes that have the specified Skyhook annotation
121	annotationKey := nodeStateAnnotationPrefix + opts.skyhookName	1✔
122	nodesToReset := make([]string, 0, len(matchedNodes))	1✔
123	nodeStates := make(map[string]v1alpha1.NodeState)	1✔
124		1✔
125	for _, nodeName := range matchedNodes {	2✔
126	idx := nodeMap[nodeName]	1✔
127	node := &nodeList.Items[idx]	1✔
128		1✔
129	annotation, ok := node.Annotations[annotationKey]	1✔
130	if !ok {	2✔
131	continue	1✔
132	}
133
134	var nodeState v1alpha1.NodeState	1✔
135	if err := json.Unmarshal([]byte(annotation), &nodeState); err != nil {	1✔
NEW 136	continue	×
137	}
138
139	nodesToReset = append(nodesToReset, nodeName)	1✔
140	nodeStates[nodeName] = nodeState	1✔
141	}
142
143	if len(nodesToReset) == 0 {	2✔
144	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "No nodes have state for Skyhook %q\n", opts.skyhookName)	1✔
145	return nil	1✔
146	}	1✔
147
148	// Print summary
149	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Skyhook: %s\n", opts.skyhookName)	1✔
150	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Nodes to reset (%d):\n", len(nodesToReset))	1✔
151	for _, nodeName := range nodesToReset {	2✔
152	nodeState := nodeStates[nodeName]	1✔
153	_, _ = fmt.Fprintf(cmd.OutOrStdout(), " - %s (%d packages)\n", nodeName, len(nodeState))	1✔
154	}	1✔
155
156	// Dry run check
157	if cliCtx.GlobalFlags.DryRun {	2✔
158	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "\n[dry-run] No changes applied\n")	1✔
159	return nil	1✔
160	}	1✔
161
162	// Confirmation
163	if !opts.confirm {	2✔
164	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nThis will remove ALL package state for Skyhook %q on these nodes.\n", opts.skyhookName)	1✔
165	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "All packages will re-run from the beginning.\n")	1✔
166	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Continue? [y/N]: ")	1✔
167		1✔
168	reader := bufio.NewReader(cmd.InOrStdin())	1✔
169	response, err := reader.ReadString('\n')	1✔
170	if err != nil {	1✔
NEW 171	return fmt.Errorf("reading confirmation: %w", err)	×
NEW 172	}	×
173
174	response = strings.ToLower(strings.TrimSpace(response))	1✔
175	if response != "y" && response != "yes" {	2✔
176	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Aborted\n")	1✔
177	return nil	1✔
178	}	1✔
179	}
180
181	// Apply changes
182	var updateErrors []string	1✔
183	successCount := 0	1✔
184		1✔
185	for _, nodeName := range nodesToReset {	2✔
186	idx := nodeMap[nodeName]	1✔
187	node := nodeList.Items[idx].DeepCopy()	1✔
188		1✔
189	// Remove the Skyhook annotation	1✔
190	delete(node.Annotations, annotationKey)	1✔
191		1✔
192	_, err := kubeClient.Kubernetes().CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{})	1✔
193	if err != nil {	1✔
NEW 194	updateErrors = append(updateErrors, fmt.Sprintf("%s: %v", nodeName, err))	×
NEW 195	continue	×
196	}
197	successCount++	1✔
198	}
199
200	// Print results
201	if len(updateErrors) > 0 {	1✔
NEW 202	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nErrors resetting some nodes:\n")	×
NEW 203	for _, e := range updateErrors {	×
NEW 204	_, _ = fmt.Fprintf(cmd.OutOrStdout(), " - %s\n", e)	×
NEW 205	}	×
206	}
207
208	if successCount > 0 {	2✔
209	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nSuccessfully reset %d node(s) for Skyhook %q\n", successCount, opts.skyhookName)	1✔
210	}	1✔
211
212	return nil	1✔
213	}

NVIDIA / skyhook / 20106791825

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous