• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

NVIDIA / skyhook / 20320280516

17 Dec 2025 11:15PM UTC coverage: 75.452% (+0.5%) from 74.903%
20320280516

push

github

web-flow
feat(cli): add package and node management commands with lifecycle controls (#123)

Add comprehensive CLI commands for managing Skyhook packages and nodes:

Package Commands:
- `package rerun`: Force re-execution of packages on specific nodes
  - Support for stage-specific re-runs (apply, config, interrupt, post-interrupt)
  - Node matching via exact names or regex patterns
- `package status`: Query package status across the cluster
- `package logs`: Retrieve package execution logs with follow/tail support

Node Commands:
- `node list`: List all nodes with Skyhook status
- `node status`: Display detailed status for specific nodes
- `node ignore`: Add/remove ignore label to pause operations on nodes
- `node reset`: Reset node state for a Skyhook

Lifecycle Commands:
- `pause`: Pause Skyhook reconciliation temporarily
- `resume`: Resume paused Skyhook operations
- `disable`: Disable a Skyhook completely
- `enable`: Re-enable a disabled Skyhook

Also includes:
- Comprehensive unit tests with K8s dynamic client mocks
- CLI e2e test suite using chainsaw (lifecycle, node, package tests)
- CI integration for CLI tests in operator-ci workflow
- Shared utilities for node matching, label management, and patch-based updates

1142 of 1535 new or added lines in 16 files covered. (74.4%)

2 existing lines in 1 file now uncovered.

5803 of 7691 relevant lines covered (75.45%)

1.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

20.77
/operator/internal/cli/utils/utils.go
1
/*
2
 * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3
 * SPDX-License-Identifier: Apache-2.0
4
 *
5
 *
6
 * Licensed under the Apache License, Version 2.0 (the "License");
7
 * you may not use this file except in compliance with the License.
8
 * You may obtain a copy of the License at
9
 *
10
 * http://www.apache.org/licenses/LICENSE-2.0
11
 *
12
 * Unless required by applicable law or agreed to in writing, software
13
 * distributed under the License is distributed on an "AS IS" BASIS,
14
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
 * See the License for the specific language governing permissions and
16
 * limitations under the License.
17
 */
18

19
package utils
20

21
import (
22
        "context"
23
        "encoding/json"
24
        "fmt"
25
        "io"
26
        "regexp"
27
        "strings"
28
        "text/tabwriter"
29

30
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31
        "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
32
        "k8s.io/apimachinery/pkg/types"
33
        "k8s.io/client-go/dynamic"
34
        "k8s.io/client-go/kubernetes"
35
        "sigs.k8s.io/yaml"
36

37
        "github.com/NVIDIA/skyhook/operator/api/v1alpha1"
38
)
39

40
// MatchNodes matches node patterns against a list of available nodes.
41
// Patterns can be exact node names or regex patterns.
42
func MatchNodes(patterns []string, availableNodes []string) ([]string, error) {
1✔
43
        matched := make(map[string]bool)
1✔
44

1✔
45
        for _, pattern := range patterns {
2✔
46
                // Check if it's a regex pattern (contains regex metacharacters)
1✔
47
                isRegex := strings.ContainsAny(pattern, ".*+?^${}[]|()\\")
1✔
48

1✔
49
                if isRegex {
2✔
50
                        re, err := regexp.Compile("^" + pattern + "$")
1✔
51
                        if err != nil {
2✔
52
                                return nil, fmt.Errorf("invalid regex pattern %q: %w", pattern, err)
1✔
53
                        }
1✔
54

55
                        for _, node := range availableNodes {
2✔
56
                                if re.MatchString(node) {
2✔
57
                                        matched[node] = true
1✔
58
                                }
1✔
59
                        }
60
                } else {
1✔
61
                        // Exact match
1✔
62
                        for _, node := range availableNodes {
2✔
63
                                if node == pattern {
2✔
64
                                        matched[node] = true
1✔
65
                                }
1✔
66
                        }
67
                }
68
        }
69

70
        result := make([]string, 0, len(matched))
1✔
71
        for node := range matched {
2✔
72
                result = append(result, node)
1✔
73
        }
1✔
74

75
        return result, nil
1✔
76
}
77

78
// EscapeJSONPointer escapes special characters in JSON Pointer tokens
79
// per RFC 6901: ~ becomes ~0, / becomes ~1
80
func EscapeJSONPointer(s string) string {
1✔
81
        s = strings.ReplaceAll(s, "~", "~0")
1✔
82
        s = strings.ReplaceAll(s, "/", "~1")
1✔
83
        return s
1✔
84
}
1✔
85

86
// UnstructuredToSkyhook converts an unstructured object to a Skyhook.
87
func UnstructuredToSkyhook(u *unstructured.Unstructured) (*v1alpha1.Skyhook, error) {
1✔
88
        data, err := u.MarshalJSON()
1✔
89
        if err != nil {
1✔
NEW
90
                return nil, fmt.Errorf("marshaling unstructured: %w", err)
×
NEW
91
        }
×
92

93
        var skyhook v1alpha1.Skyhook
1✔
94
        if err := json.Unmarshal(data, &skyhook); err != nil {
1✔
NEW
95
                return nil, fmt.Errorf("unmarshaling to skyhook: %w", err)
×
NEW
96
        }
×
97

98
        return &skyhook, nil
1✔
99
}
100

101
// Skyhook annotation and label keys
102
const (
103
        PauseAnnotation   = v1alpha1.METADATA_PREFIX + "/pause"
104
        DisableAnnotation = v1alpha1.METADATA_PREFIX + "/disable"
105
        NodeIgnoreLabel   = v1alpha1.METADATA_PREFIX + "/ignore"
106
)
107

108
// SetSkyhookAnnotation sets an annotation on a Skyhook CR using dynamic client
109
// Note: Skyhook is a cluster-scoped resource (not namespaced)
NEW
110
func SetSkyhookAnnotation(ctx context.Context, dynamicClient dynamic.Interface, skyhookName, annotation, value string) error {
×
NEW
111
        patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:%q}}}`, annotation, value)
×
NEW
112

×
NEW
113
        gvr := v1alpha1.GroupVersion.WithResource("skyhooks")
×
NEW
114
        _, err := dynamicClient.Resource(gvr).Patch(
×
NEW
115
                ctx,
×
NEW
116
                skyhookName,
×
NEW
117
                types.MergePatchType,
×
NEW
118
                []byte(patch),
×
NEW
119
                metav1.PatchOptions{},
×
NEW
120
        )
×
NEW
121
        if err != nil {
×
NEW
122
                return fmt.Errorf("patching skyhook %q: %w", skyhookName, err)
×
NEW
123
        }
×
124

NEW
125
        return nil
×
126
}
127

128
// RemoveSkyhookAnnotation removes an annotation from a Skyhook CR using dynamic client
129
// Note: Skyhook is a cluster-scoped resource (not namespaced)
NEW
130
func RemoveSkyhookAnnotation(ctx context.Context, dynamicClient dynamic.Interface, skyhookName, annotation string) error {
×
NEW
131
        patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:null}}}`, annotation)
×
NEW
132

×
NEW
133
        gvr := v1alpha1.GroupVersion.WithResource("skyhooks")
×
NEW
134
        _, err := dynamicClient.Resource(gvr).Patch(
×
NEW
135
                ctx,
×
NEW
136
                skyhookName,
×
NEW
137
                types.MergePatchType,
×
NEW
138
                []byte(patch),
×
NEW
139
                metav1.PatchOptions{},
×
NEW
140
        )
×
NEW
141
        if err != nil {
×
NEW
142
                return fmt.Errorf("patching skyhook %q: %w", skyhookName, err)
×
NEW
143
        }
×
144

NEW
145
        return nil
×
146
}
147

148
// SetNodeAnnotation sets an annotation on a Node using merge patch
NEW
149
func SetNodeAnnotation(ctx context.Context, kubeClient kubernetes.Interface, nodeName, key, value string) error {
×
NEW
150
        patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:%q}}}`, key, value)
×
NEW
151

×
NEW
152
        _, err := kubeClient.CoreV1().Nodes().Patch(
×
NEW
153
                ctx,
×
NEW
154
                nodeName,
×
NEW
155
                types.MergePatchType,
×
NEW
156
                []byte(patch),
×
NEW
157
                metav1.PatchOptions{},
×
NEW
158
        )
×
NEW
159
        if err != nil {
×
NEW
160
                return fmt.Errorf("patching node %q: %w", nodeName, err)
×
NEW
161
        }
×
162

NEW
163
        return nil
×
164
}
165

166
// RemoveNodeAnnotation removes an annotation from a Node using merge patch
NEW
167
func RemoveNodeAnnotation(ctx context.Context, kubeClient kubernetes.Interface, nodeName, key string) error {
×
NEW
168
        patch := fmt.Sprintf(`{"metadata":{"annotations":{%q:null}}}`, key)
×
NEW
169

×
NEW
170
        _, err := kubeClient.CoreV1().Nodes().Patch(
×
NEW
171
                ctx,
×
NEW
172
                nodeName,
×
NEW
173
                types.MergePatchType,
×
NEW
174
                []byte(patch),
×
NEW
175
                metav1.PatchOptions{},
×
NEW
176
        )
×
NEW
177
        if err != nil {
×
NEW
178
                return fmt.Errorf("patching node %q: %w", nodeName, err)
×
NEW
179
        }
×
180

NEW
181
        return nil
×
182
}
183

184
// SetNodeLabel sets a label on a Node using merge patch
NEW
185
func SetNodeLabel(ctx context.Context, kubeClient kubernetes.Interface, nodeName, key, value string) error {
×
NEW
186
        patch := fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, key, value)
×
NEW
187

×
NEW
188
        _, err := kubeClient.CoreV1().Nodes().Patch(
×
NEW
189
                ctx,
×
NEW
190
                nodeName,
×
NEW
191
                types.MergePatchType,
×
NEW
192
                []byte(patch),
×
NEW
193
                metav1.PatchOptions{},
×
NEW
194
        )
×
NEW
195
        if err != nil {
×
NEW
196
                return fmt.Errorf("patching node %q: %w", nodeName, err)
×
NEW
197
        }
×
198

NEW
199
        return nil
×
200
}
201

202
// RemoveNodeLabel removes a label from a Node using merge patch
NEW
203
func RemoveNodeLabel(ctx context.Context, kubeClient kubernetes.Interface, nodeName, key string) error {
×
NEW
204
        patch := fmt.Sprintf(`{"metadata":{"labels":{%q:null}}}`, key)
×
NEW
205

×
NEW
206
        _, err := kubeClient.CoreV1().Nodes().Patch(
×
NEW
207
                ctx,
×
NEW
208
                nodeName,
×
NEW
209
                types.MergePatchType,
×
NEW
210
                []byte(patch),
×
NEW
211
                metav1.PatchOptions{},
×
NEW
212
        )
×
NEW
213
        if err != nil {
×
NEW
214
                return fmt.Errorf("patching node %q: %w", nodeName, err)
×
NEW
215
        }
×
216

NEW
217
        return nil
×
218
}
219

220
// OutputJSON writes data as indented JSON to the writer
NEW
221
func OutputJSON(out io.Writer, data any) error {
×
NEW
222
        jsonData, err := json.MarshalIndent(data, "", "  ")
×
NEW
223
        if err != nil {
×
NEW
224
                return fmt.Errorf("marshaling json: %w", err)
×
NEW
225
        }
×
NEW
226
        _, _ = fmt.Fprintln(out, string(jsonData))
×
NEW
227
        return nil
×
228
}
229

230
// OutputYAML writes data as YAML to the writer
NEW
231
func OutputYAML(out io.Writer, data any) error {
×
NEW
232
        yamlData, err := yaml.Marshal(data)
×
NEW
233
        if err != nil {
×
NEW
234
                return fmt.Errorf("marshaling yaml: %w", err)
×
NEW
235
        }
×
NEW
236
        _, _ = fmt.Fprint(out, string(yamlData))
×
NEW
237
        return nil
×
238
}
239

240
// TableConfig defines the column configuration for table/wide output
241
// T is the type of items being displayed
242
type TableConfig[T any] struct {
243
        // Headers for table mode (always shown)
244
        Headers []string
245
        // Extract returns column values for table mode
246
        Extract func(T) []string
247
        // WideHeaders are additional headers appended in wide mode (optional)
248
        WideHeaders []string
249
        // WideExtract returns additional column values for wide mode (optional)
250
        WideExtract func(T) []string
251
}
252

253
// OutputTable writes items as a table using the provided config
NEW
254
func OutputTable[T any](out io.Writer, cfg TableConfig[T], items []T) error {
×
NEW
255
        return outputTableInternal(out, cfg, items, false)
×
NEW
256
}
×
257

258
// OutputWide writes items as a wide table (table columns + wide columns)
NEW
259
func OutputWide[T any](out io.Writer, cfg TableConfig[T], items []T) error {
×
NEW
260
        return outputTableInternal(out, cfg, items, true)
×
NEW
261
}
×
262

263
// OutputTableWithHeader writes items as a table with a header line above
NEW
264
func OutputTableWithHeader[T any](out io.Writer, headerLine string, cfg TableConfig[T], items []T) error {
×
NEW
265
        _, _ = fmt.Fprintln(out, headerLine)
×
NEW
266
        _, _ = fmt.Fprintln(out)
×
NEW
267
        return outputTableInternal(out, cfg, items, false)
×
NEW
268
}
×
269

270
// OutputWideWithHeader writes items as a wide table with a header line above
NEW
271
func OutputWideWithHeader[T any](out io.Writer, headerLine string, cfg TableConfig[T], items []T) error {
×
NEW
272
        _, _ = fmt.Fprintln(out, headerLine)
×
NEW
273
        _, _ = fmt.Fprintln(out)
×
NEW
274
        return outputTableInternal(out, cfg, items, true)
×
NEW
275
}
×
276

NEW
277
func outputTableInternal[T any](out io.Writer, cfg TableConfig[T], items []T, wide bool) error {
×
NEW
278
        tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
×
NEW
279

×
NEW
280
        // Build headers
×
NEW
281
        headers := cfg.Headers
×
NEW
282
        if wide && len(cfg.WideHeaders) > 0 {
×
NEW
283
                headers = append(headers, cfg.WideHeaders...)
×
NEW
284
        }
×
285

286
        // Write headers
NEW
287
        _, _ = fmt.Fprintln(tw, strings.Join(headers, "\t"))
×
NEW
288

×
NEW
289
        // Write separator
×
NEW
290
        seps := make([]string, len(headers))
×
NEW
291
        for i, h := range headers {
×
NEW
292
                seps[i] = strings.Repeat("-", len(h))
×
NEW
293
        }
×
NEW
294
        _, _ = fmt.Fprintln(tw, strings.Join(seps, "\t"))
×
NEW
295

×
NEW
296
        // Write rows
×
NEW
297
        for _, item := range items {
×
NEW
298
                row := cfg.Extract(item)
×
NEW
299
                if wide && cfg.WideExtract != nil {
×
NEW
300
                        row = append(row, cfg.WideExtract(item)...)
×
NEW
301
                }
×
NEW
302
                _, _ = fmt.Fprintln(tw, strings.Join(row, "\t"))
×
303
        }
304

NEW
305
        return tw.Flush()
×
306
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc