• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

NVIDIA / skyhook / 20150581929

11 Dec 2025 11:16PM UTC coverage: 77.153%. First build
20150581929

Pull #125

github

t0mmylam
feat: Consolidate CLI e2e tests with proper assertions and CI integration
Pull Request #125: feat: Create CLI e2e tests with assertions and CI integration

1085 of 1296 new or added lines in 13 files covered. (83.72%)

5724 of 7419 relevant lines covered (77.15%)

0.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.15
/operator/internal/cli/node/node_status.go
1
/*
2
 * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3
 * SPDX-License-Identifier: Apache-2.0
4
 *
5
 *
6
 * Licensed under the Apache License, Version 2.0 (the "License");
7
 * you may not use this file except in compliance with the License.
8
 * You may obtain a copy of the License at
9
 *
10
 * http://www.apache.org/licenses/LICENSE-2.0
11
 *
12
 * Unless required by applicable law or agreed to in writing, software
13
 * distributed under the License is distributed on an "AS IS" BASIS,
14
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
 * See the License for the specific language governing permissions and
16
 * limitations under the License.
17
 */
18

19
package node
20

21
import (
22
        "context"
23
        "encoding/json"
24
        "fmt"
25
        "io"
26
        "sort"
27
        "strings"
28
        "text/tabwriter"
29

30
        "github.com/spf13/cobra"
31
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
32
        "sigs.k8s.io/yaml"
33

34
        "github.com/NVIDIA/skyhook/operator/api/v1alpha1"
35
        "github.com/NVIDIA/skyhook/operator/internal/cli/client"
36
        cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
37
        "github.com/NVIDIA/skyhook/operator/internal/cli/utils"
38
)
39

40
const nodeStateAnnotationPrefix = v1alpha1.METADATA_PREFIX + "/nodeState_"
41

42
// nodeStatusOptions holds the options for the node status command
43
type nodeStatusOptions struct {
44
        skyhookName string
45
        output      string
46
}
47

48
// BindToCmd binds the options to the command flags
49
func (o *nodeStatusOptions) BindToCmd(cmd *cobra.Command) {
1✔
50
        cmd.Flags().StringVar(&o.skyhookName, "skyhook", "", "Filter by Skyhook name")
1✔
51
        cmd.Flags().StringVarP(&o.output, "output", "o", "table", "Output format: table, json, yaml, wide")
1✔
52
}
1✔
53

54
// NewStatusCmd creates the node status command
55
func NewStatusCmd(ctx *cliContext.CLIContext) *cobra.Command {
1✔
56
        opts := &nodeStatusOptions{}
1✔
57

1✔
58
        cmd := &cobra.Command{
1✔
59
                Use:   "status [node-name...] [flags]",
1✔
60
                Short: "Show all Skyhook activity on specific node(s)",
1✔
61
                Long: `Show all Skyhook activity on specific node(s) by reading node annotations.
1✔
62

1✔
63
This command displays a summary of all Skyhook CRs that have activity on the 
1✔
64
specified node(s), including overall status and package completion counts.
1✔
65

1✔
66
If no node name is provided, all nodes with Skyhook annotations are shown.
1✔
67
Node names can be exact matches or regex patterns.`,
1✔
68
                Example: `  # Show all Skyhook activity on a specific node
1✔
69
  kubectl skyhook node status worker-1
1✔
70

1✔
71
  # Show Skyhook activity on multiple nodes
1✔
72
  kubectl skyhook node status worker-1 worker-2 worker-3
1✔
73

1✔
74
  # Show Skyhook activity on nodes matching a pattern
1✔
75
  kubectl skyhook node status "worker-.*"
1✔
76

1✔
77
  # Filter by specific Skyhook
1✔
78
  kubectl skyhook node status worker-1 --skyhook gpu-init
1✔
79

1✔
80
  # View all nodes with Skyhook activity
1✔
81
  kubectl skyhook node status
1✔
82

1✔
83
  # Output as JSON
1✔
84
  kubectl skyhook node status worker-1 -o json
1✔
85

1✔
86
  # Output with package details
1✔
87
  kubectl skyhook node status worker-1 -o wide`,
1✔
88
                RunE: func(cmd *cobra.Command, args []string) error {
1✔
NEW
89
                        clientFactory := client.NewFactory(ctx.GlobalFlags.ConfigFlags)
×
NEW
90
                        kubeClient, err := clientFactory.Client()
×
NEW
91
                        if err != nil {
×
NEW
92
                                return fmt.Errorf("initializing kubernetes client: %w", err)
×
NEW
93
                        }
×
94

NEW
95
                        return runNodeStatus(cmd.Context(), cmd.OutOrStdout(), kubeClient, args, opts)
×
96
                },
97
        }
98

99
        opts.BindToCmd(cmd)
1✔
100

1✔
101
        return cmd
1✔
102
}
103

104
// nodeSkyhookSummary represents a summary of Skyhook activity on a node
105
type nodeSkyhookSummary struct {
106
        NodeName         string                 `json:"nodeName"`
107
        SkyhookName      string                 `json:"skyhookName"`
108
        Status           string                 `json:"status"`
109
        PackagesComplete int                    `json:"packagesComplete"`
110
        PackagesTotal    int                    `json:"packagesTotal"`
111
        Packages         []nodeSkyhookPkgStatus `json:"packages,omitempty"`
112
}
113

114
// nodeSkyhookPkgStatus represents the status of a single package
115
type nodeSkyhookPkgStatus struct {
116
        Name    string `json:"name"`
117
        Version string `json:"version"`
118
        Stage   string `json:"stage"`
119
        State   string `json:"state"`
120
        Image   string `json:"image,omitempty"`
121
}
122

123
func runNodeStatus(ctx context.Context, out io.Writer, kubeClient *client.Client, nodePatterns []string, opts *nodeStatusOptions) error {
1✔
124
        // Get all nodes
1✔
125
        nodeList, err := kubeClient.Kubernetes().CoreV1().Nodes().List(ctx, metav1.ListOptions{})
1✔
126
        if err != nil {
1✔
NEW
127
                return fmt.Errorf("listing nodes: %w", err)
×
NEW
128
        }
×
129

130
        // Collect all node names for pattern matching
131
        allNodeNames := make([]string, 0, len(nodeList.Items))
1✔
132
        for _, node := range nodeList.Items {
2✔
133
                allNodeNames = append(allNodeNames, node.Name)
1✔
134
        }
1✔
135

136
        // Filter nodes by pattern if specified
137
        var targetNodes []string
1✔
138
        if len(nodePatterns) > 0 {
2✔
139
                targetNodes, err = utils.MatchNodes(nodePatterns, allNodeNames)
1✔
140
                if err != nil {
1✔
NEW
141
                        return fmt.Errorf("matching nodes: %w", err)
×
NEW
142
                }
×
143
                if len(targetNodes) == 0 {
1✔
NEW
144
                        _, _ = fmt.Fprintf(out, "No nodes matched the specified patterns\n")
×
NEW
145
                        return nil
×
NEW
146
                }
×
147
        } else {
1✔
148
                targetNodes = allNodeNames
1✔
149
        }
1✔
150

151
        targetNodeSet := make(map[string]bool)
1✔
152
        for _, n := range targetNodes {
2✔
153
                targetNodeSet[n] = true
1✔
154
        }
1✔
155

156
        // Collect status from all nodes with Skyhook annotations
157
        var summaries []nodeSkyhookSummary
1✔
158

1✔
159
        for _, node := range nodeList.Items {
2✔
160
                if !targetNodeSet[node.Name] {
1✔
NEW
161
                        continue
×
162
                }
163

164
                // Find all Skyhook annotations on this node
165
                for annotationKey, annotationValue := range node.Annotations {
2✔
166
                        if !strings.HasPrefix(annotationKey, nodeStateAnnotationPrefix) {
1✔
NEW
167
                                continue
×
168
                        }
169

170
                        skyhookName := strings.TrimPrefix(annotationKey, nodeStateAnnotationPrefix)
1✔
171

1✔
172
                        // Filter by skyhook name if specified
1✔
173
                        if opts.skyhookName != "" && skyhookName != opts.skyhookName {
2✔
174
                                continue
1✔
175
                        }
176

177
                        var nodeState v1alpha1.NodeState
1✔
178
                        if err := json.Unmarshal([]byte(annotationValue), &nodeState); err != nil {
1✔
NEW
179
                                continue // Skip invalid annotations
×
180
                        }
181

182
                        packages := make([]nodeSkyhookPkgStatus, 0, len(nodeState))
1✔
183
                        completeCount := 0
1✔
184
                        hasError := false
1✔
185
                        hasInProgress := false
1✔
186

1✔
187
                        for _, pkgStatus := range nodeState {
2✔
188
                                packages = append(packages, nodeSkyhookPkgStatus{
1✔
189
                                        Name:    pkgStatus.Name,
1✔
190
                                        Version: pkgStatus.Version,
1✔
191
                                        Stage:   string(pkgStatus.Stage),
1✔
192
                                        State:   string(pkgStatus.State),
1✔
193
                                        Image:   pkgStatus.Image,
1✔
194
                                })
1✔
195

1✔
196
                                switch pkgStatus.State {
1✔
197
                                case v1alpha1.StateComplete:
1✔
198
                                        completeCount++
1✔
199
                                case v1alpha1.StateErroring:
1✔
200
                                        hasError = true
1✔
201
                                case v1alpha1.StateInProgress:
1✔
202
                                        hasInProgress = true
1✔
203
                                }
204
                        }
205

206
                        // Determine overall status
207
                        status := string(v1alpha1.StateUnknown)
1✔
208
                        if hasError {
2✔
209
                                status = string(v1alpha1.StateErroring)
1✔
210
                        } else if completeCount == len(packages) && len(packages) > 0 {
3✔
211
                                status = string(v1alpha1.StateComplete)
1✔
212
                        } else if hasInProgress || completeCount > 0 {
3✔
213
                                status = string(v1alpha1.StateInProgress)
1✔
214
                        }
1✔
215

216
                        // Sort packages by name
217
                        sort.Slice(packages, func(i, j int) bool {
2✔
218
                                return packages[i].Name < packages[j].Name
1✔
219
                        })
1✔
220

221
                        summaries = append(summaries, nodeSkyhookSummary{
1✔
222
                                NodeName:         node.Name,
1✔
223
                                SkyhookName:      skyhookName,
1✔
224
                                Status:           status,
1✔
225
                                PackagesComplete: completeCount,
1✔
226
                                PackagesTotal:    len(packages),
1✔
227
                                Packages:         packages,
1✔
228
                        })
1✔
229
                }
230
        }
231

232
        // Sort by node name, then skyhook name
233
        sort.Slice(summaries, func(i, j int) bool {
2✔
234
                if summaries[i].NodeName != summaries[j].NodeName {
2✔
235
                        return summaries[i].NodeName < summaries[j].NodeName
1✔
236
                }
1✔
237
                return summaries[i].SkyhookName < summaries[j].SkyhookName
1✔
238
        })
239

240
        if len(summaries) == 0 {
2✔
241
                _, _ = fmt.Fprintf(out, "No Skyhook activity found on specified nodes\n")
1✔
242
                return nil
1✔
243
        }
1✔
244

245
        // Output based on format
246
        switch opts.output {
1✔
247
        case "json":
1✔
248
                return outputNodeStatusJSON(out, summaries)
1✔
NEW
249
        case "yaml":
×
NEW
250
                return outputNodeStatusYAML(out, summaries)
×
NEW
251
        case "wide":
×
NEW
252
                return outputNodeStatusWide(out, summaries)
×
253
        default:
1✔
254
                return outputNodeStatusTable(out, summaries)
1✔
255
        }
256
}
257

258
func outputNodeStatusJSON(out io.Writer, summaries []nodeSkyhookSummary) error {
1✔
259
        data, err := json.MarshalIndent(summaries, "", "  ")
1✔
260
        if err != nil {
1✔
NEW
261
                return fmt.Errorf("marshaling json: %w", err)
×
NEW
262
        }
×
263
        _, _ = fmt.Fprintln(out, string(data))
1✔
264
        return nil
1✔
265
}
266

NEW
267
func outputNodeStatusYAML(out io.Writer, summaries []nodeSkyhookSummary) error {
×
NEW
268
        data, err := yaml.Marshal(summaries)
×
NEW
269
        if err != nil {
×
NEW
270
                return fmt.Errorf("marshaling yaml: %w", err)
×
NEW
271
        }
×
NEW
272
        _, _ = fmt.Fprint(out, string(data))
×
NEW
273
        return nil
×
274
}
275

276
func outputNodeStatusTable(out io.Writer, summaries []nodeSkyhookSummary) error {
1✔
277
        w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
1✔
278
        _, _ = fmt.Fprintln(w, "NODE\tSKYHOOK\tSTATUS\tPACKAGES-COMPLETE\tPACKAGES-TOTAL")
1✔
279
        _, _ = fmt.Fprintln(w, "----\t-------\t------\t-----------------\t--------------")
1✔
280

1✔
281
        for _, s := range summaries {
2✔
282
                _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%d/%d\t%d\n",
1✔
283
                        s.NodeName, s.SkyhookName, s.Status, s.PackagesComplete, s.PackagesTotal, s.PackagesTotal)
1✔
284
        }
1✔
285

286
        return w.Flush()
1✔
287
}
288

289
func outputNodeStatusWide(out io.Writer, summaries []nodeSkyhookSummary) error {
1✔
290
        w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
1✔
291
        _, _ = fmt.Fprintln(w, "NODE\tSKYHOOK\tPACKAGE\tVERSION\tSTAGE\tSTATE\tIMAGE")
1✔
292
        _, _ = fmt.Fprintln(w, "----\t-------\t-------\t-------\t-----\t-----\t-----")
1✔
293

1✔
294
        for _, s := range summaries {
2✔
295
                for _, pkg := range s.Packages {
2✔
296
                        _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
1✔
297
                                s.NodeName, s.SkyhookName, pkg.Name, pkg.Version, pkg.Stage, pkg.State, pkg.Image)
1✔
298
                }
1✔
299
        }
300

301
        return w.Flush()
1✔
302
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc