20320280516

Committed 17 Dec 2025 11:15PM UTC coverage: 75.452% (+0.5%) from 74.903%

Build # 20320280516

Build Type

push

github

Committed by

web-flow

Commit Message

feat(cli): add package and node management commands with lifecycle controls (#123)

Add comprehensive CLI commands for managing Skyhook packages and nodes:

Package Commands:
- `package rerun`: Force re-execution of packages on specific nodes
  - Support for stage-specific re-runs (apply, config, interrupt, post-interrupt)
  - Node matching via exact names or regex patterns
- `package status`: Query package status across the cluster
- `package logs`: Retrieve package execution logs with follow/tail support

Node Commands:
- `node list`: List all nodes with Skyhook status
- `node status`: Display detailed status for specific nodes
- `node ignore`: Add/remove ignore label to pause operations on nodes
- `node reset`: Reset node state for a Skyhook

Lifecycle Commands:
- `pause`: Pause Skyhook reconciliation temporarily
- `resume`: Resume paused Skyhook operations
- `disable`: Disable a Skyhook completely
- `enable`: Re-enable a disabled Skyhook

Also includes:
- Comprehensive unit tests with K8s dynamic client mocks
- CLI e2e test suite using chainsaw (lifecycle, node, package tests)
- CI integration for CLI tests in operator-ci workflow
- Shared utilities for node matching, label management, and patch-based updates

Run Details

1142 of 1535 new or added lines in 16 files covered. (74.4%)

2 existing lines in 1 file now uncovered.

5803 of 7691 relevant lines covered (75.45%)

1.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

54.76

/operator/internal/cli/pause.go

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cli

import (
        "fmt"

        "github.com/spf13/cobra"

        "github.com/NVIDIA/skyhook/operator/internal/cli/client"
        cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
        "github.com/NVIDIA/skyhook/operator/internal/cli/utils"
)

// pauseOptions holds the options for the pause command
type pauseOptions struct {
        confirm bool
}

// BindToCmd binds the options to the command flags
func (o *pauseOptions) BindToCmd(cmd *cobra.Command) {
        cmd.Flags().BoolVarP(&o.confirm, "confirm", "y", false, "Skip confirmation prompt")
}

// NewPauseCmd creates the pause command
func NewPauseCmd(ctx *cliContext.CLIContext) *cobra.Command {
        opts := &pauseOptions{}

        cmd := &cobra.Command{
                Use:   "pause <skyhook-name>",
                Short: "Pause a Skyhook from processing",
                Long: `Pause a Skyhook by setting the pause annotation.

When a Skyhook is paused, the operator will stop processing new nodes
but will not interrupt any currently running operations.`,
                Example: `  # Pause a Skyhook
  kubectl skyhook pause gpu-init

  # Pause without confirmation
  kubectl skyhook pause gpu-init --confirm`,
                Args: cobra.ExactArgs(1),
                RunE: func(cmd *cobra.Command, args []string) error {
                        skyhookName := args[0]

                        if !opts.confirm {
                                _, _ = fmt.Fprintf(cmd.OutOrStdout(), "This will pause Skyhook %q. Continue? [y/N]: ", skyhookName)
                                var response string
                                if _, err := fmt.Scanln(&response); err != nil || (response != "y" && response != "Y") {
                                        _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Aborted.")
                                        return nil
                                }
                        }

                        clientFactory := client.NewFactory(ctx.GlobalFlags.ConfigFlags)
                        kubeClient, err := clientFactory.Client()
                        if err != nil {
                                return fmt.Errorf("initializing kubernetes client: %w", err)
                        }

                        if err := utils.SetSkyhookAnnotation(cmd.Context(), kubeClient.Dynamic(), skyhookName, utils.PauseAnnotation, "true"); err != nil {
                                return err
                        }
                        _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Skyhook %q paused\n", skyhookName)
                        return nil
                },
        }

        opts.BindToCmd(cmd)

        return cmd
}

1	/*
2	* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3	* SPDX-License-Identifier: Apache-2.0
4	*
5	*
6	* Licensed under the Apache License, Version 2.0 (the "License");
7	* you may not use this file except in compliance with the License.
8	* You may obtain a copy of the License at
9	*
10	* http://www.apache.org/licenses/LICENSE-2.0
11	*
12	* Unless required by applicable law or agreed to in writing, software
13	* distributed under the License is distributed on an "AS IS" BASIS,
14	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	* See the License for the specific language governing permissions and
16	* limitations under the License.
17	*/
18
19	package cli
20
21	import (
22	"fmt"
23
24	"github.com/spf13/cobra"
25
26	"github.com/NVIDIA/skyhook/operator/internal/cli/client"
27	cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
28	"github.com/NVIDIA/skyhook/operator/internal/cli/utils"
29	)
30
31	// pauseOptions holds the options for the pause command
32	type pauseOptions struct {
33	confirm bool
34	}
35
36	// BindToCmd binds the options to the command flags
37	func (o pauseOptions) BindToCmd(cmd cobra.Command) {	1✔
38	cmd.Flags().BoolVarP(&o.confirm, "confirm", "y", false, "Skip confirmation prompt")	1✔
39	}	1✔
40
41	// NewPauseCmd creates the pause command
42	func NewPauseCmd(ctx cliContext.CLIContext) cobra.Command {	1✔
43	opts := &pauseOptions{}	1✔
44		1✔
45	cmd := &cobra.Command{	1✔
46	Use: "pause <skyhook-name>",	1✔
47	Short: "Pause a Skyhook from processing",	1✔
48	Long: `Pause a Skyhook by setting the pause annotation.	1✔
49		1✔
50	When a Skyhook is paused, the operator will stop processing new nodes	1✔
51	but will not interrupt any currently running operations.`,	1✔
52	Example: ` # Pause a Skyhook	1✔
53	kubectl skyhook pause gpu-init	1✔
54		1✔
55	# Pause without confirmation	1✔
56	kubectl skyhook pause gpu-init --confirm`,	1✔
57	Args: cobra.ExactArgs(1),	1✔
58	RunE: func(cmd *cobra.Command, args []string) error {	1✔
NEW 59	skyhookName := args[0]	×
NEW 60		×
NEW 61	if !opts.confirm {	×
NEW 62	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "This will pause Skyhook %q. Continue? [y/N]: ", skyhookName)	×
NEW 63	var response string	×
NEW 64	if _, err := fmt.Scanln(&response); err != nil \|\| (response != "y" && response != "Y") {	×
NEW 65	_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Aborted.")	×
NEW 66	return nil	×
NEW 67	}	×
68	}
69
NEW 70	clientFactory := client.NewFactory(ctx.GlobalFlags.ConfigFlags)	×
NEW 71	kubeClient, err := clientFactory.Client()	×
NEW 72	if err != nil {	×
NEW 73	return fmt.Errorf("initializing kubernetes client: %w", err)	×
NEW 74	}	×
75
NEW 76	if err := utils.SetSkyhookAnnotation(cmd.Context(), kubeClient.Dynamic(), skyhookName, utils.PauseAnnotation, "true"); err != nil {	×
NEW 77	return err	×
NEW 78	}	×
NEW 79	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Skyhook %q paused\n", skyhookName)	×
NEW 80	return nil	×
81	},
82	}
83
84	opts.BindToCmd(cmd)	1✔
85		1✔
86	return cmd	1✔
87	}

NVIDIA / skyhook / 20320280516

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous