6222

Committed 27 Jan 2022 12:44PM UTC coverage: 57.347%. First build

Build # 6222

Build Type

push

travis-ci-com

Committed by

akhilamohanan

Commit Message

<a href="https://github.com/noironetworks/aci-containers/commit/<a class=hub.com/noironetworks/aci-containers/commit/faa59478bf8780dd8e1a03b0483d711c591342a8">faa59478b<a href="https://github.com/noironetworks/aci-containers/commit/faa59478bf8780dd8e1a03b0483d711c591342a8">&quot;&gt;Fixed deadlock in hostagent

Trying to access snatPolicyCacheMutex.RLock again in same goroutine
without releasing already acquired snatPolicyCacheMutex.RLock was causing
deadlock. This goroutine accessed its first snatPolicyCacheMutex.RLock
and then another goroutine stated waiting for snatPolicyCacheMutex.Lock
and hence the first goroutine was not able to access the 2nd
snatPolicyCacheMutex.RLock which caused the deadlock

Avoided the 2nd occurance of snatPolicyCacheMutex.RLock in same
goroutine without releasing the 1st read lock.

(cherry picked from commit </a><a class="double-link" href="https://github.com/noironetworks/aci-containers/commit/<a class="double-link" href="https://github.com/noironetworks/aci-containers/commit/65e7351d01252b1f561af892bccc538cc7c3e586">65e7351d0</a>">65e7351d0</a><a href="https://github.com/noironetworks/aci-containers/commit/faa59478bf8780dd8e1a03b0483d711c591342a8">)

Run Details

3 of 3 new or added lines in 2 files covered. (100.0%)

11572 of 20179 relevant lines covered (57.35%)

0.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

7.61

/pkg/hostagent/snatlocalinfo.go

// Copyright 2019 Cisco Systems, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRATIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// creates snat crs.

package hostagent

import (
        "context"
        "reflect"

        snatLocalInfov1 "github.com/noironetworks/aci-containers/pkg/snatlocalinfo/apis/aci.snat/v1"
        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type SnatLocalInfo struct {
        snatIp         string
        destIps        []string
        snatpolicyName string
}

func (agent *HostAgent) UpdateLocalInfoCr() bool {
        env := agent.env.(*K8sEnvironment)
        snatLocalInfoClient := env.snatLocalInfoClient
        if snatLocalInfoClient == nil {
                agent.log.Debug("snatLocalInfo or Kube clients are not intialized")
                return false
        }
        agent.indexMutex.Lock()
        ginfos, ok := agent.opflexSnatGlobalInfos[agent.config.NodeName]
        if !ok {
                agent.indexMutex.Unlock()
                return agent.deleteLocalInfoCr()
        }

        snatLocalInfo := make(map[string]SnatLocalInfo)
        for _, ginfo := range ginfos {
                var localInfo SnatLocalInfo
                localInfo.snatIp = ginfo.SnatIp
                agent.snatPolicyCacheMutex.RLock()
                if _, ok := agent.snatPolicyCache[ginfo.SnatPolicyName]; ok {
                        if len(agent.snatPolicyCache[ginfo.SnatPolicyName].Spec.DestIp) == 0 {
                                localInfo.destIps = []string{"0.0.0.0/0"}
                        } else {
                                localInfo.destIps =
                                        agent.snatPolicyCache[ginfo.SnatPolicyName].Spec.DestIp
                        }
                }
                agent.snatPolicyCacheMutex.RUnlock()
                localInfo.snatpolicyName = ginfo.SnatPolicyName
                snatLocalInfo[ginfo.SnatIpUid] = localInfo
        }
        var localInfos []snatLocalInfov1.LocalInfo
        for uid, v := range agent.opflexSnatLocalInfos {
                var localinfo snatLocalInfov1.LocalInfo
                var policies []snatLocalInfov1.SnatPolicy
                for _, plcyUid := range v.PlcyUuids {
                        if linfo, ok := snatLocalInfo[plcyUid]; ok {
                                var policy snatLocalInfov1.SnatPolicy
                                policy.SnatIp = linfo.snatIp
                                policy.DestIp = linfo.destIps
                                policy.Name = linfo.snatpolicyName
                                policies = append(policies, policy)
                        }
                }
                epkey := agent.podUidToName[uid]
                podobj, exists, err := agent.podInformer.GetStore().GetByKey(epkey)
                if exists && err == nil {
                        pod := podobj.(*v1.Pod)
                        localinfo.PodName = pod.ObjectMeta.Name
                        localinfo.PodNamespace = pod.ObjectMeta.Namespace
                        agent.log.Debugf("Adding pod %s to snatLocalInfo:", localinfo.PodName)
                }
                localinfo.PodUid = uid
                localinfo.SnatPolicies = policies
                if len(policies) > 0 {
                        localInfos = append(localInfos, localinfo)
                }
        }
        agent.indexMutex.Unlock()
        snatLocalInfoCr, err := snatLocalInfoClient.AciV1().SnatLocalInfos(agent.config.AciSnatNamespace).Get(context.TODO(), agent.config.NodeName, metav1.GetOptions{})
        if err != nil {
                if apierrors.IsNotFound(err) {
                        snatLocalInfoInstance := &snatLocalInfov1.SnatLocalInfo{
                                ObjectMeta: metav1.ObjectMeta{
                                        Name:      agent.config.NodeName,
                                        Namespace: agent.config.AciSnatNamespace,
                                },
                                Spec: snatLocalInfov1.SnatLocalInfoSpec{
                                        LocalInfos: localInfos,
                                },
                        }
                        _, err = snatLocalInfoClient.AciV1().SnatLocalInfos(agent.config.AciSnatNamespace).Create(context.TODO(), snatLocalInfoInstance, metav1.CreateOptions{})
                }
        } else {
                Spec := snatLocalInfov1.SnatLocalInfoSpec{
                        LocalInfos: localInfos,
                }
                if !reflect.DeepEqual(snatLocalInfoCr.Spec, Spec) {
                        snatLocalInfoCr.Spec = Spec
                        _, err = snatLocalInfoClient.AciV1().SnatLocalInfos(agent.config.AciSnatNamespace).Update(context.TODO(), snatLocalInfoCr, metav1.UpdateOptions{})
                }
        }
        if err == nil {
                agent.log.Debug("SnatLocalInfo  Update Successful..")
                return false
        }
        return true
}

func (agent *HostAgent) deleteLocalInfoCr() bool {
        env := agent.env.(*K8sEnvironment)
        snatLocalInfoClient := env.snatLocalInfoClient
        _, err := snatLocalInfoClient.AciV1().SnatLocalInfos(agent.config.AciSnatNamespace).Get(context.TODO(), agent.config.NodeName, metav1.GetOptions{})
        if err == nil {
                err = snatLocalInfoClient.AciV1().SnatLocalInfos(agent.config.AciSnatNamespace).Delete(context.TODO(), agent.config.NodeName, metav1.DeleteOptions{})
                if err != nil {
                        agent.log.Debug("SnatLocalInfo failed to delete: ", err)
                        return true
                }
        }
        return false
}

1	// Copyright 2019 Cisco Systems, Inc.
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRATIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	// creates snat crs.
16
17	package hostagent
18
19	import (
20	"context"
21	"reflect"
22
23	snatLocalInfov1 "github.com/noironetworks/aci-containers/pkg/snatlocalinfo/apis/aci.snat/v1"
24	v1 "k8s.io/api/core/v1"
25	apierrors "k8s.io/apimachinery/pkg/api/errors"
26	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27	)
28
29	type SnatLocalInfo struct {
30	snatIp string
31	destIps []string
32	snatpolicyName string
33	}
34
35	func (agent *HostAgent) UpdateLocalInfoCr() bool {	1✔
36	env := agent.env.(*K8sEnvironment)	1✔
37	snatLocalInfoClient := env.snatLocalInfoClient	1✔
38	if snatLocalInfoClient == nil {	2✔
39	agent.log.Debug("snatLocalInfo or Kube clients are not intialized")	1✔
40	return false	1✔
41	}	1✔
42	agent.indexMutex.Lock()	×
43	ginfos, ok := agent.opflexSnatGlobalInfos[agent.config.NodeName]	×
44	if !ok {	×
45	agent.indexMutex.Unlock()	×
46	return agent.deleteLocalInfoCr()	×
47	}	×
48
49	snatLocalInfo := make(map[string]SnatLocalInfo)	×
50	for _, ginfo := range ginfos {	×
51	var localInfo SnatLocalInfo	×
52	localInfo.snatIp = ginfo.SnatIp	×
53	agent.snatPolicyCacheMutex.RLock()	×
54	if _, ok := agent.snatPolicyCache[ginfo.SnatPolicyName]; ok {	×
55	if len(agent.snatPolicyCache[ginfo.SnatPolicyName].Spec.DestIp) == 0 {	×
56	localInfo.destIps = []string{"0.0.0.0/0"}	×
57	} else {	×
58	localInfo.destIps =	×
59	agent.snatPolicyCache[ginfo.SnatPolicyName].Spec.DestIp	×
60	}	×
61	}
62	agent.snatPolicyCacheMutex.RUnlock()	×
63	localInfo.snatpolicyName = ginfo.SnatPolicyName	×
64	snatLocalInfo[ginfo.SnatIpUid] = localInfo	×
65	}
66	var localInfos []snatLocalInfov1.LocalInfo	×
67	for uid, v := range agent.opflexSnatLocalInfos {	×
68	var localinfo snatLocalInfov1.LocalInfo	×
69	var policies []snatLocalInfov1.SnatPolicy	×
70	for _, plcyUid := range v.PlcyUuids {	×
71	if linfo, ok := snatLocalInfo[plcyUid]; ok {	×
72	var policy snatLocalInfov1.SnatPolicy	×
73	policy.SnatIp = linfo.snatIp	×
74	policy.DestIp = linfo.destIps	×
75	policy.Name = linfo.snatpolicyName	×
76	policies = append(policies, policy)	×
77	}	×
78	}
79	epkey := agent.podUidToName[uid]	×
80	podobj, exists, err := agent.podInformer.GetStore().GetByKey(epkey)	×
81	if exists && err == nil {	×
82	pod := podobj.(*v1.Pod)	×
83	localinfo.PodName = pod.ObjectMeta.Name	×
84	localinfo.PodNamespace = pod.ObjectMeta.Namespace	×
85	agent.log.Debugf("Adding pod %s to snatLocalInfo:", localinfo.PodName)	×
86	}	×
87	localinfo.PodUid = uid	×
88	localinfo.SnatPolicies = policies	×
89	if len(policies) > 0 {	×
90	localInfos = append(localInfos, localinfo)	×
91	}	×
92	}
93	agent.indexMutex.Unlock()	×
94	snatLocalInfoCr, err := snatLocalInfoClient.AciV1().SnatLocalInfos(agent.config.AciSnatNamespace).Get(context.TODO(), agent.config.NodeName, metav1.GetOptions{})	×
95	if err != nil {	×
96	if apierrors.IsNotFound(err) {	×
97	snatLocalInfoInstance := &snatLocalInfov1.SnatLocalInfo{	×
98	ObjectMeta: metav1.ObjectMeta{	×
99	Name: agent.config.NodeName,	×
100	Namespace: agent.config.AciSnatNamespace,	×
101	},	×
102	Spec: snatLocalInfov1.SnatLocalInfoSpec{	×
103	LocalInfos: localInfos,	×
104	},	×
105	}	×
106	_, err = snatLocalInfoClient.AciV1().SnatLocalInfos(agent.config.AciSnatNamespace).Create(context.TODO(), snatLocalInfoInstance, metav1.CreateOptions{})	×
107	}	×
108	} else {	×
109	Spec := snatLocalInfov1.SnatLocalInfoSpec{	×
110	LocalInfos: localInfos,	×
111	}	×
112	if !reflect.DeepEqual(snatLocalInfoCr.Spec, Spec) {	×
113	snatLocalInfoCr.Spec = Spec	×
114	_, err = snatLocalInfoClient.AciV1().SnatLocalInfos(agent.config.AciSnatNamespace).Update(context.TODO(), snatLocalInfoCr, metav1.UpdateOptions{})	×
115	}	×
116	}
117	if err == nil {	×
118	agent.log.Debug("SnatLocalInfo Update Successful..")	×
119	return false	×
120	}	×
121	return true	×
122	}
123
124	func (agent *HostAgent) deleteLocalInfoCr() bool {	×
125	env := agent.env.(*K8sEnvironment)	×
126	snatLocalInfoClient := env.snatLocalInfoClient	×
127	_, err := snatLocalInfoClient.AciV1().SnatLocalInfos(agent.config.AciSnatNamespace).Get(context.TODO(), agent.config.NodeName, metav1.GetOptions{})	×
128	if err == nil {	×
129	err = snatLocalInfoClient.AciV1().SnatLocalInfos(agent.config.AciSnatNamespace).Delete(context.TODO(), agent.config.NodeName, metav1.DeleteOptions{})	×
130	if err != nil {	×
131	agent.log.Debug("SnatLocalInfo failed to delete: ", err)	×
132	return true	×
133	}	×
134	}
135	return false	×
136	}

noironetworks / aci-containers / 6222

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous