19227470377

Committed 10 Nov 2025 09:48AM UTC coverage: 62.151% (-0.2%) from 62.366%

Build # 19227470377

Build Type

Pull #902

github

Committed by

web-flow

Commit Message

Merge f9637c189 into 3d1a472a6

Pull Request Pull Request #902: Create platform and orchestrator packages

Run Details

319 of 659 new or added lines in 25 files covered. (48.41%)

41 existing lines in 9 files now uncovered.

8772 of 14114 relevant lines covered (62.15%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.85

/pkg/plugins/k8s/k8s_plugin.go

package k8s

import (
        "strings"

        "sigs.k8s.io/controller-runtime/pkg/log"

        sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
        hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
        plugins "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
        "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
)

var PluginName = "k8s"

type K8sPlugin struct {
        PluginName  string
        SpecVersion string
        hostHelper  helper.HostHelpersInterface

        openVSwitchService      *hostTypes.Service
        sriovService            *hostTypes.Service
        sriovPostNetworkService *hostTypes.Service

        updateTarget *k8sUpdateTarget
}
type updateTargetReq struct {
        update bool
        reboot bool
}

// set need update flag for updateTargetReq
func (u *updateTargetReq) SetNeedUpdate() {
        u.update = true
}

// set need update and reboot flags for updateTargetReq
func (u *updateTargetReq) SetNeedReboot() {
        u.update = true
        u.reboot = true
}

// returns state of the update flag
func (u *updateTargetReq) NeedUpdate() bool {
        return u.update
}

// returns state of the reboot flag
func (u *updateTargetReq) NeedReboot() bool {
        return u.reboot
}

type k8sUpdateTarget struct {
        sriovScript            updateTargetReq
        sriovPostNetworkScript updateTargetReq
        openVSwitch            updateTargetReq
}

func (u *k8sUpdateTarget) String() string {
        var updateList []string
        if u.sriovScript.NeedReboot() {
                updateList = append(updateList, "sriov-config.service")
        }
        if u.sriovPostNetworkScript.NeedReboot() {
                updateList = append(updateList, "sriov-config-post-network.service")
        }
        if u.openVSwitch.NeedReboot() {
                updateList = append(updateList, "ovs-vswitchd.service")
        }
        return strings.Join(updateList, ",")
}

func (u *k8sUpdateTarget) needReboot() bool {
        return u.sriovScript.NeedReboot() || u.sriovPostNetworkScript.NeedReboot() || u.openVSwitch.NeedReboot()
}

func (u *k8sUpdateTarget) reset() {
        u.sriovScript = updateTargetReq{}
        u.sriovPostNetworkScript = updateTargetReq{}
        u.openVSwitch = updateTargetReq{}
}

const (
        bindataManifestPath      = "bindata/manifests/"
        switchdevManifestPath    = bindataManifestPath + "switchdev-config/"
        switchdevUnits           = switchdevManifestPath + "switchdev-units/"
        sriovUnits               = bindataManifestPath + "sriov-config-service/kubernetes/"
        sriovUnitFile            = sriovUnits + "sriov-config-service.yaml"
        sriovPostNetworkUnitFile = sriovUnits + "sriov-config-post-network-service.yaml"
        ovsUnitFile              = switchdevManifestPath + "ovs-units/ovs-vswitchd.service.yaml"
)

// Initialize our plugin and set up initial values
func NewK8sPlugin(helper helper.HostHelpersInterface) (plugins.VendorPlugin, error) {
        k8sPluging := &K8sPlugin{
                PluginName:   PluginName,
                hostHelper:   helper,
                updateTarget: &k8sUpdateTarget{},
        }

        return k8sPluging, k8sPluging.readManifestFiles()
}

// Name returns the name of the plugin
func (p *K8sPlugin) Name() string {
        return p.PluginName
}

// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node
func (p *K8sPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {
        log.Log.Info("k8s plugin OnNodeStateChange()")
        needDrain = false
        needReboot = false

        p.updateTarget.reset()
        // TODO add check for enableOvsOffload in OperatorConfig later
        // Update services if switchdev required
        if !vars.UsingSystemdMode && !sriovnetworkv1.IsSwitchdevModeSpec(new.Spec) {
                return
        }

        if sriovnetworkv1.IsSwitchdevModeSpec(new.Spec) {
                // Check services
                err = p.ovsServiceStateUpdate()
                if err != nil {
                        log.Log.Error(err, "k8s plugin OnNodeStateChange(): failed")
                        return
                }
        }

        if vars.UsingSystemdMode {
                // Check sriov service
                err = p.sriovServicesStateUpdate()
                if err != nil {
                        log.Log.Error(err, "k8s plugin OnNodeStateChange(): failed")
                        return
                }
        }

        if p.updateTarget.needReboot() {
                needDrain = true
                needReboot = true
                log.Log.Info("k8s plugin OnNodeStateChange(): needReboot to update", "target", p.updateTarget)
        }

        return
}

// TODO: implement - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/630
// OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs.
func (p *K8sPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) (bool, error) {
        return false, nil
}

// Apply config change
func (p *K8sPlugin) Apply() error {
        log.Log.Info("k8s plugin Apply()")
        if vars.UsingSystemdMode {
                if err := p.updateSriovServices(); err != nil {
                        return err
                }
        }
        return p.updateOVSService()
}

func (p *K8sPlugin) readOpenVSwitchdManifest() error {
        openVSwitchService, err := p.hostHelper.ReadServiceInjectionManifestFile(ovsUnitFile)
        if err != nil {
                return err
        }
        p.openVSwitchService = openVSwitchService
        return nil
}

func (p *K8sPlugin) readSriovServiceManifest() error {
        sriovService, err := p.hostHelper.ReadServiceManifestFile(sriovUnitFile)
        if err != nil {
                return err
        }
        p.sriovService = sriovService
        return nil
}

func (p *K8sPlugin) readSriovPostNetworkServiceManifest() error {
        sriovService, err := p.hostHelper.ReadServiceManifestFile(sriovPostNetworkUnitFile)
        if err != nil {
                return err
        }
        p.sriovPostNetworkService = sriovService
        return nil
}

func (p *K8sPlugin) readManifestFiles() error {
        if err := p.readOpenVSwitchdManifest(); err != nil {
                return err
        }
        if err := p.readSriovServiceManifest(); err != nil {
                return err
        }
        if err := p.readSriovPostNetworkServiceManifest(); err != nil {
                return err
        }
        return nil
}

func (p *K8sPlugin) sriovServicesStateUpdate() error {
        for _, s := range []struct {
                srv    *hostTypes.Service
                update *updateTargetReq
        }{
                {srv: p.sriovService, update: &p.updateTarget.sriovScript},
                {srv: p.sriovPostNetworkService, update: &p.updateTarget.sriovPostNetworkScript},
        } {
                isServiceEnabled, err := p.hostHelper.IsServiceEnabled(s.srv.Path)
                if err != nil {
                        return err
                }
                // create and enable the service if it doesn't exist or is not enabled
                if !isServiceEnabled {
                        s.update.SetNeedReboot()
                } else {
                        if p.isSystemDServiceNeedUpdate(s.srv) {
                                s.update.SetNeedReboot()
                        }
                }
        }
        return nil
}

func (p *K8sPlugin) updateSriovServices() error {
        for _, s := range []struct {
                srv    *hostTypes.Service
                update *updateTargetReq
        }{
                {srv: p.sriovService, update: &p.updateTarget.sriovScript},
                {srv: p.sriovPostNetworkService, update: &p.updateTarget.sriovPostNetworkScript},
        } {
                if s.update.NeedUpdate() {
                        err := p.hostHelper.EnableService(s.srv)
                        if err != nil {
                                return err
                        }
                }
        }
        return nil
}

func (p *K8sPlugin) ovsServiceStateUpdate() error {
        exist, err := p.hostHelper.IsServiceExist(p.openVSwitchService.Path)
        if err != nil {
                return err
        }
        if !exist {
                log.Log.Info("k8s plugin systemServicesStateUpdate(): WARNING! openvswitch system service not found, skip update",
                        "service", p.openVSwitchService.Name)
                return nil
        }
        if !p.isSystemDServiceNeedUpdate(p.openVSwitchService) {
                // service is up to date
                return nil
        }
        if p.isOVSHwOffloadingEnabled() {
                p.updateTarget.openVSwitch.SetNeedUpdate()
        } else {
                p.updateTarget.openVSwitch.SetNeedReboot()
        }
        return nil
}

func (p *K8sPlugin) updateOVSService() error {
        if p.updateTarget.openVSwitch.NeedUpdate() {
                return p.hostHelper.UpdateSystemService(p.openVSwitchService)
        }
        return nil
}

func (p *K8sPlugin) isSystemDServiceNeedUpdate(serviceObj *hostTypes.Service) bool {
        systemService, err := p.hostHelper.ReadService(serviceObj.Path)
        if err != nil {
                log.Log.Error(err, "k8s plugin isSystemDServiceNeedUpdate(): failed to read service file, ignoring",
                        "path", serviceObj.Path)
                return false
        }
        if systemService != nil {
                needChange, err := p.hostHelper.CompareServices(systemService, serviceObj)
                if err != nil {
                        log.Log.Error(err, "k8s plugin isSystemDServiceNeedUpdate(): failed to compare service, ignoring")
                        return false
                }
                return needChange
        }
        return false
}

// try to check if OVS HW offloading is already enabled
// required to avoid unneeded reboots in case if HW offloading is already enabled by different entity
// TODO move to the right package and avoid ovs-vsctl binary call
// the function should be revisited when support for software bridge configuration
// is implemented
func (p *K8sPlugin) isOVSHwOffloadingEnabled() bool {
        log.Log.V(2).Info("isOVSHwOffloadingEnabled()")
        exit, err := p.hostHelper.Chroot(consts.Chroot)
        if err != nil {
                return false
        }
        defer exit()
        out, _, err := p.hostHelper.RunCommand("ovs-vsctl", "get", "Open_vSwitch", ".", "other_config:hw-offload")
        if err != nil {
                log.Log.V(2).Info("isOVSHwOffloadingEnabled() check failed, assume offloading is disabled", "error", err.Error())
                return false
        }
        if strings.Trim(out, "\n") == `"true"` {
                log.Log.V(2).Info("isOVSHwOffloadingEnabled() offloading is already enabled")
                return true
        }
        return false
}

1	package k8s
2
3	import (
4	"strings"
5
6	"sigs.k8s.io/controller-runtime/pkg/log"
7
8	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
9	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
10	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper"
11	hostTypes "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types"
12	plugins "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins"
13	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
14	)
15
16	var PluginName = "k8s"
17
18	type K8sPlugin struct {
19	PluginName string
20	SpecVersion string
21	hostHelper helper.HostHelpersInterface
22
23	openVSwitchService *hostTypes.Service
24	sriovService *hostTypes.Service
25	sriovPostNetworkService *hostTypes.Service
26
27	updateTarget *k8sUpdateTarget
28	}
29	type updateTargetReq struct {
30	update bool
31	reboot bool
32	}
33
34	// set need update flag for updateTargetReq
35	func (u *updateTargetReq) SetNeedUpdate() {	1✔
36	u.update = true	1✔
37	}	1✔
38
39	// set need update and reboot flags for updateTargetReq
40	func (u *updateTargetReq) SetNeedReboot() {	1✔
41	u.update = true	1✔
42	u.reboot = true	1✔
43	}	1✔
44
45	// returns state of the update flag
46	func (u *updateTargetReq) NeedUpdate() bool {	1✔
47	return u.update	1✔
48	}	1✔
49
50	// returns state of the reboot flag
51	func (u *updateTargetReq) NeedReboot() bool {	1✔
52	return u.reboot	1✔
53	}	1✔
54
55	type k8sUpdateTarget struct {
56	sriovScript updateTargetReq
57	sriovPostNetworkScript updateTargetReq
58	openVSwitch updateTargetReq
59	}
60
61	func (u *k8sUpdateTarget) String() string {	1✔
62	var updateList []string	1✔
63	if u.sriovScript.NeedReboot() {	2✔
64	updateList = append(updateList, "sriov-config.service")	1✔
65	}	1✔
66	if u.sriovPostNetworkScript.NeedReboot() {	2✔
67	updateList = append(updateList, "sriov-config-post-network.service")	1✔
68	}	1✔
69	if u.openVSwitch.NeedReboot() {	2✔
70	updateList = append(updateList, "ovs-vswitchd.service")	1✔
71	}	1✔
72	return strings.Join(updateList, ",")	1✔
73	}
74
75	func (u *k8sUpdateTarget) needReboot() bool {	1✔
76	return u.sriovScript.NeedReboot() \|\| u.sriovPostNetworkScript.NeedReboot() \|\| u.openVSwitch.NeedReboot()	1✔
77	}	1✔
78
79	func (u *k8sUpdateTarget) reset() {	1✔
80	u.sriovScript = updateTargetReq{}	1✔
81	u.sriovPostNetworkScript = updateTargetReq{}	1✔
82	u.openVSwitch = updateTargetReq{}	1✔
83	}	1✔
84
85	const (
86	bindataManifestPath = "bindata/manifests/"
87	switchdevManifestPath = bindataManifestPath + "switchdev-config/"
88	switchdevUnits = switchdevManifestPath + "switchdev-units/"
89	sriovUnits = bindataManifestPath + "sriov-config-service/kubernetes/"
90	sriovUnitFile = sriovUnits + "sriov-config-service.yaml"
91	sriovPostNetworkUnitFile = sriovUnits + "sriov-config-post-network-service.yaml"
92	ovsUnitFile = switchdevManifestPath + "ovs-units/ovs-vswitchd.service.yaml"
93	)
94
95	// Initialize our plugin and set up initial values
96	func NewK8sPlugin(helper helper.HostHelpersInterface) (plugins.VendorPlugin, error) {	1✔
97	k8sPluging := &K8sPlugin{	1✔
98	PluginName: PluginName,	1✔
99	hostHelper: helper,	1✔
100	updateTarget: &k8sUpdateTarget{},	1✔
101	}	1✔
102		1✔
103	return k8sPluging, k8sPluging.readManifestFiles()	1✔
104	}	1✔
105
106	// Name returns the name of the plugin
107	func (p *K8sPlugin) Name() string {	1✔
108	return p.PluginName	1✔
109	}	1✔
110
111	// OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node
112	func (p K8sPlugin) OnNodeStateChange(new sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) {	1✔
113	log.Log.Info("k8s plugin OnNodeStateChange()")	1✔
114	needDrain = false	1✔
115	needReboot = false	1✔
116		1✔
117	p.updateTarget.reset()	1✔
118	// TODO add check for enableOvsOffload in OperatorConfig later	1✔
119	// Update services if switchdev required	1✔
120	if !vars.UsingSystemdMode && !sriovnetworkv1.IsSwitchdevModeSpec(new.Spec) {	2✔
121	return	1✔
122	}	1✔
123
124	if sriovnetworkv1.IsSwitchdevModeSpec(new.Spec) {	2✔
125	// Check services	1✔
126	err = p.ovsServiceStateUpdate()	1✔
127	if err != nil {	1✔
128	log.Log.Error(err, "k8s plugin OnNodeStateChange(): failed")	×
129	return	×
130	}	×
131	}
132
133	if vars.UsingSystemdMode {	2✔
134	// Check sriov service	1✔
135	err = p.sriovServicesStateUpdate()	1✔
136	if err != nil {	1✔
137	log.Log.Error(err, "k8s plugin OnNodeStateChange(): failed")	×
138	return	×
139	}	×
140	}
141
142	if p.updateTarget.needReboot() {	2✔
143	needDrain = true	1✔
144	needReboot = true	1✔
145	log.Log.Info("k8s plugin OnNodeStateChange(): needReboot to update", "target", p.updateTarget)	1✔
146	}	1✔
147
148	return	1✔
149	}
150
151	// TODO: implement - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/630
152	// OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs.
UNCOV 153	func (p K8sPlugin) CheckStatusChanges(sriovnetworkv1.SriovNetworkNodeState) (bool, error) {	×
UNCOV 154	return false, nil	×
UNCOV 155	}	×
156
157	// Apply config change
158	func (p *K8sPlugin) Apply() error {	1✔
159	log.Log.Info("k8s plugin Apply()")	1✔
160	if vars.UsingSystemdMode {	2✔
161	if err := p.updateSriovServices(); err != nil {	1✔
162	return err	×
163	}	×
164	}
165	return p.updateOVSService()	1✔
166	}
167
168	func (p *K8sPlugin) readOpenVSwitchdManifest() error {	1✔
169	openVSwitchService, err := p.hostHelper.ReadServiceInjectionManifestFile(ovsUnitFile)	1✔
170	if err != nil {	1✔
171	return err	×
172	}	×
173	p.openVSwitchService = openVSwitchService	1✔
174	return nil	1✔
175	}
176
177	func (p *K8sPlugin) readSriovServiceManifest() error {	1✔
178	sriovService, err := p.hostHelper.ReadServiceManifestFile(sriovUnitFile)	1✔
179	if err != nil {	1✔
180	return err	×
181	}	×
182	p.sriovService = sriovService	1✔
183	return nil	1✔
184	}
185
186	func (p *K8sPlugin) readSriovPostNetworkServiceManifest() error {	1✔
187	sriovService, err := p.hostHelper.ReadServiceManifestFile(sriovPostNetworkUnitFile)	1✔
188	if err != nil {	1✔
189	return err	×
190	}	×
191	p.sriovPostNetworkService = sriovService	1✔
192	return nil	1✔
193	}
194
195	func (p *K8sPlugin) readManifestFiles() error {	1✔
196	if err := p.readOpenVSwitchdManifest(); err != nil {	1✔
197	return err	×
198	}	×
199	if err := p.readSriovServiceManifest(); err != nil {	1✔
200	return err	×
201	}	×
202	if err := p.readSriovPostNetworkServiceManifest(); err != nil {	1✔
203	return err	×
204	}	×
205	return nil	1✔
206	}
207
208	func (p *K8sPlugin) sriovServicesStateUpdate() error {	1✔
209	for _, s := range []struct {	1✔
210	srv *hostTypes.Service	1✔
211	update *updateTargetReq	1✔
212	}{	1✔
213	{srv: p.sriovService, update: &p.updateTarget.sriovScript},	1✔
214	{srv: p.sriovPostNetworkService, update: &p.updateTarget.sriovPostNetworkScript},	1✔
215	} {	2✔
216	isServiceEnabled, err := p.hostHelper.IsServiceEnabled(s.srv.Path)	1✔
217	if err != nil {	1✔
218	return err	×
219	}	×
220	// create and enable the service if it doesn't exist or is not enabled
221	if !isServiceEnabled {	2✔
222	s.update.SetNeedReboot()	1✔
223	} else {	2✔
224	if p.isSystemDServiceNeedUpdate(s.srv) {	2✔
225	s.update.SetNeedReboot()	1✔
226	}	1✔
227	}
228	}
229	return nil	1✔
230	}
231
232	func (p *K8sPlugin) updateSriovServices() error {	1✔
233	for _, s := range []struct {	1✔
234	srv *hostTypes.Service	1✔
235	update *updateTargetReq	1✔
236	}{	1✔
237	{srv: p.sriovService, update: &p.updateTarget.sriovScript},	1✔
238	{srv: p.sriovPostNetworkService, update: &p.updateTarget.sriovPostNetworkScript},	1✔
239	} {	2✔
240	if s.update.NeedUpdate() {	2✔
241	err := p.hostHelper.EnableService(s.srv)	1✔
242	if err != nil {	1✔
243	return err	×
244	}	×
245	}
246	}
247	return nil	1✔
248	}
249
250	func (p *K8sPlugin) ovsServiceStateUpdate() error {	1✔
251	exist, err := p.hostHelper.IsServiceExist(p.openVSwitchService.Path)	1✔
252	if err != nil {	1✔
253	return err	×
254	}	×
255	if !exist {	2✔
256	log.Log.Info("k8s plugin systemServicesStateUpdate(): WARNING! openvswitch system service not found, skip update",	1✔
257	"service", p.openVSwitchService.Name)	1✔
258	return nil	1✔
259	}	1✔
260	if !p.isSystemDServiceNeedUpdate(p.openVSwitchService) {	1✔
261	// service is up to date	×
262	return nil	×
263	}	×
264	if p.isOVSHwOffloadingEnabled() {	2✔
265	p.updateTarget.openVSwitch.SetNeedUpdate()	1✔
266	} else {	2✔
267	p.updateTarget.openVSwitch.SetNeedReboot()	1✔
268	}	1✔
269	return nil	1✔
270	}
271
272	func (p *K8sPlugin) updateOVSService() error {	1✔
273	if p.updateTarget.openVSwitch.NeedUpdate() {	2✔
274	return p.hostHelper.UpdateSystemService(p.openVSwitchService)	1✔
275	}	1✔
276	return nil	1✔
277	}
278
279	func (p K8sPlugin) isSystemDServiceNeedUpdate(serviceObj hostTypes.Service) bool {	1✔
280	systemService, err := p.hostHelper.ReadService(serviceObj.Path)	1✔
281	if err != nil {	1✔
282	log.Log.Error(err, "k8s plugin isSystemDServiceNeedUpdate(): failed to read service file, ignoring",	×
283	"path", serviceObj.Path)	×
284	return false	×
285	}	×
286	if systemService != nil {	2✔
287	needChange, err := p.hostHelper.CompareServices(systemService, serviceObj)	1✔
288	if err != nil {	1✔
289	log.Log.Error(err, "k8s plugin isSystemDServiceNeedUpdate(): failed to compare service, ignoring")	×
290	return false	×
291	}	×
292	return needChange	1✔
293	}
294	return false	×
295	}
296
297	// try to check if OVS HW offloading is already enabled
298	// required to avoid unneeded reboots in case if HW offloading is already enabled by different entity
299	// TODO move to the right package and avoid ovs-vsctl binary call
300	// the function should be revisited when support for software bridge configuration
301	// is implemented
302	func (p *K8sPlugin) isOVSHwOffloadingEnabled() bool {	1✔
303	log.Log.V(2).Info("isOVSHwOffloadingEnabled()")	1✔
304	exit, err := p.hostHelper.Chroot(consts.Chroot)	1✔
305	if err != nil {	2✔
306	return false	1✔
307	}	1✔
308	defer exit()	1✔
309	out, _, err := p.hostHelper.RunCommand("ovs-vsctl", "get", "Open_vSwitch", ".", "other_config:hw-offload")	1✔
310	if err != nil {	1✔
311	log.Log.V(2).Info("isOVSHwOffloadingEnabled() check failed, assume offloading is disabled", "error", err.Error())	×
312	return false	×
313	}	×
314	if strings.Trim(out, "\n") == `"true"` {	2✔
315	log.Log.V(2).Info("isOVSHwOffloadingEnabled() offloading is already enabled")	1✔
316	return true	1✔
317	}	1✔
318	return false	×
319	}

k8snetworkplumbingwg / sriov-network-operator / 19227470377

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous