20072025419

Committed 09 Dec 2025 05:02PM UTC coverage: 77.041% (+2.1%) from 74.963%

Build # 20072025419

Build Type

Pull #124

github

Committed by

t0mmylam

Commit Message

feat(cli): Add node management commands and ignore label support

Pull Request Pull Request #124: feat(cli): Add node management commands and ignore label support

Run Details

1059 of 1277 new or added lines in 13 files covered. (82.93%)

56 existing lines in 3 files now uncovered.

5701 of 7400 relevant lines covered (77.04%)

0.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.89

/operator/internal/cli/node/node_status.go

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package node

import (
        "context"
        "encoding/json"
        "fmt"
        "io"
        "sort"
        "strings"
        "text/tabwriter"

        "github.com/spf13/cobra"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "sigs.k8s.io/yaml"

        "github.com/NVIDIA/skyhook/operator/api/v1alpha1"
        "github.com/NVIDIA/skyhook/operator/internal/cli/client"
        cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
        "github.com/NVIDIA/skyhook/operator/internal/cli/utils"
)

const nodeStateAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/nodeState_"

// nodeStatusOptions holds the options for the node status command
type nodeStatusOptions struct {
        skyhookName string
        output      string
}

// NewStatusCmd creates the node status command
func NewStatusCmd(ctx *cliContext.CLIContext) *cobra.Command {
        opts := &nodeStatusOptions{}

        cmd := &cobra.Command{
                Use:   "status [node-name...] [flags]",
                Short: "Show all Skyhook activity on specific node(s)",
                Long: `Show all Skyhook activity on specific node(s) by reading node annotations.

This command displays a summary of all Skyhook CRs that have activity on the 
specified node(s), including overall status and package completion counts.

If no node name is provided, all nodes with Skyhook annotations are shown.
Node names can be exact matches or regex patterns.`,
                Example: `  # Show all Skyhook activity on a specific node
  kubectl skyhook node status worker-1

  # Show Skyhook activity on multiple nodes
  kubectl skyhook node status worker-1 worker-2 worker-3

  # Show Skyhook activity on nodes matching a pattern
  kubectl skyhook node status "worker-.*"

  # Filter by specific Skyhook
  kubectl skyhook node status worker-1 --skyhook gpu-init

  # View all nodes with Skyhook activity
  kubectl skyhook node status

  # Output as JSON
  kubectl skyhook node status worker-1 -o json

  # Output with package details
  kubectl skyhook node status worker-1 -o wide`,
                RunE: func(cmd *cobra.Command, args []string) error {
                        clientFactory := client.NewFactory(ctx.GlobalFlags.ConfigFlags)
                        kubeClient, err := clientFactory.Client()
                        if err != nil {
                                return fmt.Errorf("initializing kubernetes client: %w", err)
                        }

                        return runNodeStatus(cmd.Context(), cmd.OutOrStdout(), kubeClient, args, opts)
                },
        }

        cmd.Flags().StringVar(&opts.skyhookName, "skyhook", "", "Filter by Skyhook name")
        cmd.Flags().StringVarP(&opts.output, "output", "o", "table", "Output format: table, json, yaml, wide")

        return cmd
}

// nodeSkyhookSummary represents a summary of Skyhook activity on a node
type nodeSkyhookSummary struct {
        NodeName         string                 `json:"nodeName"`
        SkyhookName      string                 `json:"skyhookName"`
        Status           string                 `json:"status"`
        PackagesComplete int                    `json:"packagesComplete"`
        PackagesTotal    int                    `json:"packagesTotal"`
        Packages         []nodeSkyhookPkgStatus `json:"packages,omitempty"`
}

// nodeSkyhookPkgStatus represents the status of a single package
type nodeSkyhookPkgStatus struct {
        Name    string `json:"name"`
        Version string `json:"version"`
        Stage   string `json:"stage"`
        State   string `json:"state"`
        Image   string `json:"image,omitempty"`
}

func runNodeStatus(ctx context.Context, out io.Writer, kubeClient *client.Client, nodePatterns []string, opts *nodeStatusOptions) error {
        // Get all nodes
        nodeList, err := kubeClient.Kubernetes().CoreV1().Nodes().List(ctx, metav1.ListOptions{})
        if err != nil {
                return fmt.Errorf("listing nodes: %w", err)
        }

        // Collect all node names for pattern matching
        allNodeNames := make([]string, 0, len(nodeList.Items))
        for _, node := range nodeList.Items {
                allNodeNames = append(allNodeNames, node.Name)
        }

        // Filter nodes by pattern if specified
        var targetNodes []string
        if len(nodePatterns) > 0 {
                targetNodes, err = utils.MatchNodes(nodePatterns, allNodeNames)
                if err != nil {
                        return fmt.Errorf("matching nodes: %w", err)
                }
                if len(targetNodes) == 0 {
                        _, _ = fmt.Fprintf(out, "No nodes matched the specified patterns\n")
                        return nil
                }
        } else {
                targetNodes = allNodeNames
        }

        targetNodeSet := make(map[string]bool)
        for _, n := range targetNodes {
                targetNodeSet[n] = true
        }

        // Collect status from all nodes with Skyhook annotations
        var summaries []nodeSkyhookSummary

        for _, node := range nodeList.Items {
                if !targetNodeSet[node.Name] {
                        continue
                }

                // Find all Skyhook annotations on this node
                for annotationKey, annotationValue := range node.Annotations {
                        if !strings.HasPrefix(annotationKey, nodeStateAnnotationPrefix) {
                                continue
                        }

                        skyhookName := strings.TrimPrefix(annotationKey, nodeStateAnnotationPrefix)

                        // Filter by skyhook name if specified
                        if opts.skyhookName != "" && skyhookName != opts.skyhookName {
                                continue
                        }

                        var nodeState v1alpha1.NodeState
                        if err := json.Unmarshal([]byte(annotationValue), &nodeState); err != nil {
                                continue // Skip invalid annotations
                        }

                        packages := make([]nodeSkyhookPkgStatus, 0, len(nodeState))
                        completeCount := 0
                        hasError := false
                        hasInProgress := false

                        for _, pkgStatus := range nodeState {
                                packages = append(packages, nodeSkyhookPkgStatus{
                                        Name:    pkgStatus.Name,
                                        Version: pkgStatus.Version,
                                        Stage:   string(pkgStatus.Stage),
                                        State:   string(pkgStatus.State),
                                        Image:   pkgStatus.Image,
                                })

                                switch pkgStatus.State {
                                case v1alpha1.StateComplete:
                                        completeCount++
                                case v1alpha1.StateErroring:
                                        hasError = true
                                case v1alpha1.StateInProgress:
                                        hasInProgress = true
                                }
                        }

                        // Determine overall status
                        status := string(v1alpha1.StateUnknown)
                        if hasError {
                                status = string(v1alpha1.StateErroring)
                        } else if completeCount == len(packages) && len(packages) > 0 {
                                status = string(v1alpha1.StateComplete)
                        } else if hasInProgress || completeCount > 0 {
                                status = string(v1alpha1.StateInProgress)
                        }

                        // Sort packages by name
                        sort.Slice(packages, func(i, j int) bool {
                                return packages[i].Name < packages[j].Name
                        })

                        summaries = append(summaries, nodeSkyhookSummary{
                                NodeName:         node.Name,
                                SkyhookName:      skyhookName,
                                Status:           status,
                                PackagesComplete: completeCount,
                                PackagesTotal:    len(packages),
                                Packages:         packages,
                        })
                }
        }

        // Sort by node name, then skyhook name
        sort.Slice(summaries, func(i, j int) bool {
                if summaries[i].NodeName != summaries[j].NodeName {
                        return summaries[i].NodeName < summaries[j].NodeName
                }
                return summaries[i].SkyhookName < summaries[j].SkyhookName
        })

        if len(summaries) == 0 {
                _, _ = fmt.Fprintf(out, "No Skyhook activity found on specified nodes\n")
                return nil
        }

        // Output based on format
        switch opts.output {
        case "json":
                return outputNodeStatusJSON(out, summaries)
        case "yaml":
                return outputNodeStatusYAML(out, summaries)
        case "wide":
                return outputNodeStatusWide(out, summaries)
        default:
                return outputNodeStatusTable(out, summaries)
        }
}

func outputNodeStatusJSON(out io.Writer, summaries []nodeSkyhookSummary) error {
        data, err := json.MarshalIndent(summaries, "", "  ")
        if err != nil {
                return fmt.Errorf("marshaling json: %w", err)
        }
        _, _ = fmt.Fprintln(out, string(data))
        return nil
}

func outputNodeStatusYAML(out io.Writer, summaries []nodeSkyhookSummary) error {
        data, err := yaml.Marshal(summaries)
        if err != nil {
                return fmt.Errorf("marshaling yaml: %w", err)
        }
        _, _ = fmt.Fprint(out, string(data))
        return nil
}

func outputNodeStatusTable(out io.Writer, summaries []nodeSkyhookSummary) error {
        w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
        _, _ = fmt.Fprintln(w, "NODE\tSKYHOOK\tSTATUS\tPACKAGES-COMPLETE\tPACKAGES-TOTAL")
        _, _ = fmt.Fprintln(w, "----\t-------\t------\t-----------------\t--------------")

        for _, s := range summaries {
                _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%d/%d\t%d\n",
                        s.NodeName, s.SkyhookName, s.Status, s.PackagesComplete, s.PackagesTotal, s.PackagesTotal)
        }

        return w.Flush()
}

func outputNodeStatusWide(out io.Writer, summaries []nodeSkyhookSummary) error {
        w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
        _, _ = fmt.Fprintln(w, "NODE\tSKYHOOK\tPACKAGE\tVERSION\tSTAGE\tSTATE\tIMAGE")
        _, _ = fmt.Fprintln(w, "----\t-------\t-------\t-------\t-----\t-----\t-----")

        for _, s := range summaries {
                for _, pkg := range s.Packages {
                        _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
                                s.NodeName, s.SkyhookName, pkg.Name, pkg.Version, pkg.Stage, pkg.State, pkg.Image)
                }
        }

        return w.Flush()
}

1	/*
2	* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3	* SPDX-License-Identifier: Apache-2.0
4	*
5	*
6	* Licensed under the Apache License, Version 2.0 (the "License");
7	* you may not use this file except in compliance with the License.
8	* You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing, software
13	* distributed under the License is distributed on an "AS IS" BASIS,
14	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	* See the License for the specific language governing permissions and
16	* limitations under the License.
17	*/
18
19	package node
20
21	import (
22	"context"
23	"encoding/json"
24	"fmt"
25	"io"
26	"sort"
27	"strings"
28	"text/tabwriter"
29
30	"github.com/spf13/cobra"
31	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
32	"sigs.k8s.io/yaml"
33
34	"github.com/NVIDIA/skyhook/operator/api/v1alpha1"
35	"github.com/NVIDIA/skyhook/operator/internal/cli/client"
36	cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
37	"github.com/NVIDIA/skyhook/operator/internal/cli/utils"
38	)
39
40	const nodeStateAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/nodeState_"
41
42	// nodeStatusOptions holds the options for the node status command
43	type nodeStatusOptions struct {
44	skyhookName string
45	output string
46	}
47
48	// NewStatusCmd creates the node status command
49	func NewStatusCmd(ctx cliContext.CLIContext) cobra.Command {	1✔
50	opts := &nodeStatusOptions{}	1✔
51		1✔
52	cmd := &cobra.Command{	1✔
53	Use: "status [node-name...] [flags]",	1✔
54	Short: "Show all Skyhook activity on specific node(s)",	1✔
55	Long: `Show all Skyhook activity on specific node(s) by reading node annotations.	1✔
56		1✔
57	This command displays a summary of all Skyhook CRs that have activity on the	1✔
58	specified node(s), including overall status and package completion counts.	1✔
59		1✔
60	If no node name is provided, all nodes with Skyhook annotations are shown.	1✔
61	Node names can be exact matches or regex patterns.`,	1✔
62	Example: ` # Show all Skyhook activity on a specific node	1✔
63	kubectl skyhook node status worker-1	1✔
64		1✔
65	# Show Skyhook activity on multiple nodes	1✔
66	kubectl skyhook node status worker-1 worker-2 worker-3	1✔
67		1✔
68	# Show Skyhook activity on nodes matching a pattern	1✔
69	kubectl skyhook node status "worker-.*"	1✔
70		1✔
71	# Filter by specific Skyhook	1✔
72	kubectl skyhook node status worker-1 --skyhook gpu-init	1✔
73		1✔
74	# View all nodes with Skyhook activity	1✔
75	kubectl skyhook node status	1✔
76		1✔
77	# Output as JSON	1✔
78	kubectl skyhook node status worker-1 -o json	1✔
79		1✔
80	# Output with package details	1✔
81	kubectl skyhook node status worker-1 -o wide`,	1✔
82	RunE: func(cmd *cobra.Command, args []string) error {	1✔
NEW 83	clientFactory := client.NewFactory(ctx.GlobalFlags.ConfigFlags)	×
NEW 84	kubeClient, err := clientFactory.Client()	×
NEW 85	if err != nil {	×
NEW 86	return fmt.Errorf("initializing kubernetes client: %w", err)	×
NEW 87	}	×
88
NEW 89	return runNodeStatus(cmd.Context(), cmd.OutOrStdout(), kubeClient, args, opts)	×
90	},
91	}
92
93	cmd.Flags().StringVar(&opts.skyhookName, "skyhook", "", "Filter by Skyhook name")	1✔
94	cmd.Flags().StringVarP(&opts.output, "output", "o", "table", "Output format: table, json, yaml, wide")	1✔
95		1✔
96	return cmd	1✔
97	}
98
99	// nodeSkyhookSummary represents a summary of Skyhook activity on a node
100	type nodeSkyhookSummary struct {
101	NodeName string `json:"nodeName"`
102	SkyhookName string `json:"skyhookName"`
103	Status string `json:"status"`
104	PackagesComplete int `json:"packagesComplete"`
105	PackagesTotal int `json:"packagesTotal"`
106	Packages []nodeSkyhookPkgStatus `json:"packages,omitempty"`
107	}
108
109	// nodeSkyhookPkgStatus represents the status of a single package
110	type nodeSkyhookPkgStatus struct {
111	Name string `json:"name"`
112	Version string `json:"version"`
113	Stage string `json:"stage"`
114	State string `json:"state"`
115	Image string `json:"image,omitempty"`
116	}
117
118	func runNodeStatus(ctx context.Context, out io.Writer, kubeClient client.Client, nodePatterns []string, opts nodeStatusOptions) error {	1✔
119	// Get all nodes	1✔
120	nodeList, err := kubeClient.Kubernetes().CoreV1().Nodes().List(ctx, metav1.ListOptions{})	1✔
121	if err != nil {	1✔
NEW 122	return fmt.Errorf("listing nodes: %w", err)	×
NEW 123	}	×
124
125	// Collect all node names for pattern matching
126	allNodeNames := make([]string, 0, len(nodeList.Items))	1✔
127	for _, node := range nodeList.Items {	2✔
128	allNodeNames = append(allNodeNames, node.Name)	1✔
129	}	1✔
130
131	// Filter nodes by pattern if specified
132	var targetNodes []string	1✔
133	if len(nodePatterns) > 0 {	2✔
134	targetNodes, err = utils.MatchNodes(nodePatterns, allNodeNames)	1✔
135	if err != nil {	1✔
NEW 136	return fmt.Errorf("matching nodes: %w", err)	×
NEW 137	}	×
138	if len(targetNodes) == 0 {	1✔
NEW 139	_, _ = fmt.Fprintf(out, "No nodes matched the specified patterns\n")	×
NEW 140	return nil	×
NEW 141	}	×
142	} else {	1✔
143	targetNodes = allNodeNames	1✔
144	}	1✔
145
146	targetNodeSet := make(map[string]bool)	1✔
147	for _, n := range targetNodes {	2✔
148	targetNodeSet[n] = true	1✔
149	}	1✔
150
151	// Collect status from all nodes with Skyhook annotations
152	var summaries []nodeSkyhookSummary	1✔
153		1✔
154	for _, node := range nodeList.Items {	2✔
155	if !targetNodeSet[node.Name] {	1✔
NEW 156	continue	×
157	}
158
159	// Find all Skyhook annotations on this node
160	for annotationKey, annotationValue := range node.Annotations {	2✔
161	if !strings.HasPrefix(annotationKey, nodeStateAnnotationPrefix) {	1✔
NEW 162	continue	×
163	}
164
165	skyhookName := strings.TrimPrefix(annotationKey, nodeStateAnnotationPrefix)	1✔
166		1✔
167	// Filter by skyhook name if specified	1✔
168	if opts.skyhookName != "" && skyhookName != opts.skyhookName {	2✔
169	continue	1✔
170	}
171
172	var nodeState v1alpha1.NodeState	1✔
173	if err := json.Unmarshal([]byte(annotationValue), &nodeState); err != nil {	1✔
NEW 174	continue // Skip invalid annotations	×
175	}
176
177	packages := make([]nodeSkyhookPkgStatus, 0, len(nodeState))	1✔
178	completeCount := 0	1✔
179	hasError := false	1✔
180	hasInProgress := false	1✔
181		1✔
182	for _, pkgStatus := range nodeState {	2✔
183	packages = append(packages, nodeSkyhookPkgStatus{	1✔
184	Name: pkgStatus.Name,	1✔
185	Version: pkgStatus.Version,	1✔
186	Stage: string(pkgStatus.Stage),	1✔
187	State: string(pkgStatus.State),	1✔
188	Image: pkgStatus.Image,	1✔
189	})	1✔
190		1✔
191	switch pkgStatus.State {	1✔
192	case v1alpha1.StateComplete:	1✔
193	completeCount++	1✔
194	case v1alpha1.StateErroring:	1✔
195	hasError = true	1✔
196	case v1alpha1.StateInProgress:	1✔
197	hasInProgress = true	1✔
198	}
199	}
200
201	// Determine overall status
202	status := string(v1alpha1.StateUnknown)	1✔
203	if hasError {	2✔
204	status = string(v1alpha1.StateErroring)	1✔
205	} else if completeCount == len(packages) && len(packages) > 0 {	3✔
206	status = string(v1alpha1.StateComplete)	1✔
207	} else if hasInProgress \|\| completeCount > 0 {	3✔
208	status = string(v1alpha1.StateInProgress)	1✔
209	}	1✔
210
211	// Sort packages by name
212	sort.Slice(packages, func(i, j int) bool {	2✔
213	return packages[i].Name < packages[j].Name	1✔
214	})	1✔
215
216	summaries = append(summaries, nodeSkyhookSummary{	1✔
217	NodeName: node.Name,	1✔
218	SkyhookName: skyhookName,	1✔
219	Status: status,	1✔
220	PackagesComplete: completeCount,	1✔
221	PackagesTotal: len(packages),	1✔
222	Packages: packages,	1✔
223	})	1✔
224	}
225	}
226
227	// Sort by node name, then skyhook name
228	sort.Slice(summaries, func(i, j int) bool {	2✔
229	if summaries[i].NodeName != summaries[j].NodeName {	2✔
230	return summaries[i].NodeName < summaries[j].NodeName	1✔
231	}	1✔
232	return summaries[i].SkyhookName < summaries[j].SkyhookName	1✔
233	})
234
235	if len(summaries) == 0 {	2✔
236	_, _ = fmt.Fprintf(out, "No Skyhook activity found on specified nodes\n")	1✔
237	return nil	1✔
238	}	1✔
239
240	// Output based on format
241	switch opts.output {	1✔
242	case "json":	1✔
243	return outputNodeStatusJSON(out, summaries)	1✔
NEW 244	case "yaml":	×
NEW 245	return outputNodeStatusYAML(out, summaries)	×
NEW 246	case "wide":	×
NEW 247	return outputNodeStatusWide(out, summaries)	×
248	default:	1✔
249	return outputNodeStatusTable(out, summaries)	1✔
250	}
251	}
252
253	func outputNodeStatusJSON(out io.Writer, summaries []nodeSkyhookSummary) error {	1✔
254	data, err := json.MarshalIndent(summaries, "", " ")	1✔
255	if err != nil {	1✔
NEW 256	return fmt.Errorf("marshaling json: %w", err)	×
NEW 257	}	×
258	_, _ = fmt.Fprintln(out, string(data))	1✔
259	return nil	1✔
260	}
261
NEW 262	func outputNodeStatusYAML(out io.Writer, summaries []nodeSkyhookSummary) error {	×
NEW 263	data, err := yaml.Marshal(summaries)	×
NEW 264	if err != nil {	×
NEW 265	return fmt.Errorf("marshaling yaml: %w", err)	×
NEW 266	}	×
NEW 267	_, _ = fmt.Fprint(out, string(data))	×
NEW 268	return nil	×
269	}
270
271	func outputNodeStatusTable(out io.Writer, summaries []nodeSkyhookSummary) error {	1✔
272	w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)	1✔
273	_, _ = fmt.Fprintln(w, "NODE\tSKYHOOK\tSTATUS\tPACKAGES-COMPLETE\tPACKAGES-TOTAL")	1✔
274	_, _ = fmt.Fprintln(w, "----\t-------\t------\t-----------------\t--------------")	1✔
275		1✔
276	for _, s := range summaries {	2✔
277	_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%d/%d\t%d\n",	1✔
278	s.NodeName, s.SkyhookName, s.Status, s.PackagesComplete, s.PackagesTotal, s.PackagesTotal)	1✔
279	}	1✔
280
281	return w.Flush()	1✔
282	}
283
284	func outputNodeStatusWide(out io.Writer, summaries []nodeSkyhookSummary) error {	1✔
285	w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)	1✔
286	_, _ = fmt.Fprintln(w, "NODE\tSKYHOOK\tPACKAGE\tVERSION\tSTAGE\tSTATE\tIMAGE")	1✔
287	_, _ = fmt.Fprintln(w, "----\t-------\t-------\t-------\t-----\t-----\t-----")	1✔
288		1✔
289	for _, s := range summaries {	2✔
290	for _, pkg := range s.Packages {	2✔
291	_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",	1✔
292	s.NodeName, s.SkyhookName, pkg.Name, pkg.Version, pkg.Stage, pkg.State, pkg.Image)	1✔
293	}	1✔
294	}
295
296	return w.Flush()	1✔
297	}

NVIDIA / skyhook / 20072025419

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous