noironetworks / aci-containers / 11070

coverage: 65.889% (-0.06%) from 65.945%
11070

push

travis-pro

web-flow 
Merge pull request #1580 from noironetworks/fixing_vmm_lite_cdets

Added fixes for CDETS

0 of 107 new or added lines in 2 files covered. (0.0%)

2 existing lines in 1 file now uncovered.

13359 of 20275 relevant lines covered (65.89%)

0.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/pkg/controller/aaepmonitor.go
1 
// Copyright 2019 Cisco Systems, Inc.
2 
//
3 
// Licensed under the Apache License, Version 2.0 (the "License");
4 
// you may not use this file except in compliance with the License.
5 
// You may obtain a copy of the License at
6 
//
7 
//     http://www.apache.org/licenses/LICENSE-2.0
8 
//
9 
// Unless required by applicable law or agreed to in writing, software
10 
// distributed under the License is distributed on an "AS IS" BASIS,
11 
// WITHOUT WARRATIES OR CONDITIONS OF ANY KIND, either express or implied.
12 
// See the License for the specific language governing permissions and
13 
// limitations under the License.
14 









15



// Handlers for AaepMonitor CR updates.










16












17



package controller










18












19



import (










20



        "context"










21



        "crypto/sha256"










22



        "encoding/hex"










23



        "encoding/json"










24



        "fmt"










25



        "regexp"










26



        "strconv"










27



        "strings"










28












29



        amv1 "github.com/noironetworks/aci-containers/pkg/aaepmonitor/apis/aci.attachmentmonitor/v1"










30



        aaepmonitorclientset "github.com/noironetworks/aci-containers/pkg/aaepmonitor/clientset/versioned"










31



        "github.com/noironetworks/aci-containers/pkg/apicapi"










32












33



        nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"










34



        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"










35



        "k8s.io/apimachinery/pkg/runtime"










36



        "k8s.io/apimachinery/pkg/watch"










37



        "k8s.io/client-go/tools/cache"










38



        "k8s.io/kubernetes/pkg/controller"










39



)










40












41



const (










42



        aaepMonitorCRDName = "aaepmonitors.aci.attachmentmonitor"










43



)










44












45



func (cont *AciController) queueAaepMonitorConfigByKey(key string) {




×







46



        cont.aaepMonitorConfigQueue.Add(key)




×







47



}




×







48












49



func aaepMonitorInit(cont *AciController, stopCh <-chan struct{}) {




×







50



        restconfig := cont.env.RESTConfig()




×







51



        aaepMonitorClient, err := aaepmonitorclientset.NewForConfig(restconfig)




×







52



        if err != nil {




×







53



                cont.log.Errorf("Failed to intialize aaepMonitorClient")




×







54



                return




×







55



        }




×







56












57



        cont.initAaepMonitorInformerFromClient(aaepMonitorClient)




×







58



        go cont.aaepMonitorInformer.Run(stopCh)




×







59



        go cont.processQueue(cont.aaepMonitorConfigQueue, cont.aaepMonitorInformer.GetIndexer(),




×







60



                func(obj interface{}) bool {




×







61



                        return cont.handleAaepMonitorConfigurationUpdate(obj)




×







62



                }, func(key string) bool {




×







63



                        return cont.handleAaepMonitorConfigurationDelete(key)




×







64



                }, nil, stopCh)




×







65



        cache.WaitForCacheSync(stopCh, cont.aaepMonitorInformer.HasSynced)




×







66



}










67












68



func (cont *AciController) initAaepMonitorInformerFromClient(










69



        aaepMonitorClient *aaepmonitorclientset.Clientset) {




×







70



        cont.initAaepMonitorInformerBase(




×







71



                &cache.ListWatch{




×







72



                        ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {




×







73



                                return aaepMonitorClient.AciV1().AaepMonitors().List(context.TODO(), options)




×







74



                        },




×







75



                        WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {




×







76



                                return aaepMonitorClient.AciV1().AaepMonitors().Watch(context.TODO(), options)




×







77



                        },




×







78



                })










79



}










80












81



func (cont *AciController) initAaepMonitorInformerBase(listWatch *cache.ListWatch) {




×







82



        cont.aaepMonitorInformer = cache.NewSharedIndexInformer(




×







83



                listWatch,




×







84



                &amv1.AaepMonitor{},




×







85



                controller.NoResyncPeriodFunc(),




×







86



                cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},




×







87



        )




×







88



        cont.aaepMonitorInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{




×







89



                AddFunc: func(obj interface{}) {




×







90



                        cont.aaepMonitorConfAdded(obj)




×







91



                },




×







92



                UpdateFunc: func(oldobj interface{}, newobj interface{}) {




×







93



                        cont.aaepMonitorConfUpdate(oldobj, newobj)




×







94



                },




×







95



                DeleteFunc: func(obj interface{}) {




×







96



                        cont.aaepMonitorConfDelete(obj)




×







97



                },




×







98



        })










99



}










100












101



func (cont *AciController) aaepMonitorConfAdded(obj interface{}) {




×







102



        aaepMonitorConfig, ok := obj.(*amv1.AaepMonitor)




×







103



        if !ok {




×







104



                cont.log.Error("aaepMonitorConfAdded: Bad object type")




×







105



                return




×







106



        }




×







107



        key, err := cache.MetaNamespaceKeyFunc(aaepMonitorConfig)




×







108



        if err != nil {




×







109



                return




×







110



        }




×







111



        cont.queueAaepMonitorConfigByKey(key)




×







112



}










113












114



func (cont *AciController) aaepMonitorConfUpdate(oldobj interface{}, newobj interface{}) {




×







115



        newAaepMonitorConfig := newobj.(*amv1.AaepMonitor)




×







116






×







117



        key, err := cache.MetaNamespaceKeyFunc(newAaepMonitorConfig)




×







118



        if err != nil {




×







119



                return




×







120



        }




×







121



        cont.queueAaepMonitorConfigByKey(key)




×







122



}










123












124



func (cont *AciController) aaepMonitorConfDelete(obj interface{}) {




×







125



        cont.indexMutex.Lock()




×







126



        defer cont.indexMutex.Unlock()




×







127



        aaepMonitorConfig, ok := obj.(*amv1.AaepMonitor)




×







128






×







129



        if !ok {




×







130



                deletedState, ok := obj.(cache.DeletedFinalStateUnknown)




×







131



                if !ok {




×







132



                        cont.log.Errorf("Received unexpected object: ")




×







133



                        return




×







134



                }




×







135



                aaepMonitorConfig, ok = deletedState.Obj.(*amv1.AaepMonitor)




×







136



                if !ok {




×







137



                        cont.log.Errorf("DeletedFinalStateUnknown contained non-aaepmonitorconfiguration object: %v", deletedState.Obj)




×







138



                        return




×







139



                }




×







140



        }










141












142



        key, err := cache.MetaNamespaceKeyFunc(aaepMonitorConfig)




×







143



        if err != nil {




×







144



                return




×







145



        }




×







146



        cont.queueAaepMonitorConfigByKey("DELETED_" + key)




×







147



}










148












149



func (cont *AciController) handleAaepMonitorConfigurationUpdate(obj interface{}) bool {




×







150



        aaepMonitorConfig, ok := obj.(*amv1.AaepMonitor)




×







151



        if !ok {




×







152



                cont.log.Error("handleAaepMonitorConfigurationUpdate: Bad object type")




×







153



                return false




×







154



        }




×







155












156



        addedAaeps, removedAaeps := cont.getAaepDiff(aaepMonitorConfig.Spec.Aaeps)




×







157



        for _, aaepName := range addedAaeps {




×







158



                cont.reconcileNadData(aaepName)




×







159



        }




×







160












161



        for _, aaepName := range removedAaeps {




×







162



                cont.cleanAnnotationSubscriptions(aaepName)




×







163






×







164



                cont.indexMutex.Lock()




×







165



                aaepEpgDataList := cont.sharedAaepMonitor[aaepName]




×







166



                delete(cont.sharedAaepMonitor, aaepName)




×







167






×







168



                for _, aaepEpgData := range aaepEpgDataList {




×








NEW


169



                        cont.deleteNetworkAttachmentDefinition(aaepName, aaepEpgData, "AaepRemovedFromCR")




×







170



                }




×







171



                cont.indexMutex.Unlock()




×







172



        }










173












174



        return false




×







175



}










176












177



func (cont *AciController) handleAaepMonitorConfigurationDelete(key string) bool {




×







178



        cont.indexMutex.Lock()




×







179



        defer cont.indexMutex.Unlock()




×







180



        for aaepName, aaepMonitorDataList := range cont.sharedAaepMonitor {




×







181



                cont.cleanAnnotationSubscriptions(aaepName)




×







182






×







183



                for _, aaepMonitorData := range aaepMonitorDataList {




×








NEW


184



                        cont.deleteNetworkAttachmentDefinition(aaepName, aaepMonitorData, "CRDeleted")




×







185



                }




×







186



        }










187












188



        cont.sharedAaepMonitor = make(map[string][]*AaepMonitoringData)




×







189



        return false




×







190



}










191












192



func (cont *AciController) handleAaepEpgAttach(infraRsObj apicapi.ApicObject) {




×







193



        infraRsObjDn := infraRsObj.GetDn()




×







194



        aaepName := cont.matchesAEPFilter(infraRsObjDn)




×







195



        if aaepName == "" {




×







196



                cont.log.Debugf("Unable to find AAEP from %s in monitoring list", infraRsObjDn)




×







197



                return




×







198



        }




×







199












200



        state := infraRsObj.GetAttrStr("state")




×







201



        if state != "formed" {




×








NEW


202



                cont.log.Debugf("Skipping NAD creation: %s is with state: %s", infraRsObjDn, state)




×







203



                return




×







204



        }




×







205












206



        epgDn := infraRsObj.GetAttrStr("tDn")




×







207






×







208



        defer cont.apicConn.AddImmediateSubscriptionDnLocked(epgDn, []string{"tagAnnotation"}, cont.handleAnnotationAdded,




×







209



                cont.handleAnnotationDeleted)




×







210






×







211



        encap := infraRsObj.GetAttrStr("encap")




×







212



        vlanID := cont.getVlanId(encap)




×







213






×







214



        aaepEpgData := &AaepEpgAttachData{




×







215



                epgDn:     epgDn,




×







216



                encapVlan: vlanID,




×







217



        }




×







218






×







219



        aaepMonitorData := cont.collectNadData(aaepEpgData)




×







220



        if aaepMonitorData == nil {




×







221



                return




×







222



        }




×







223












224



        cont.indexMutex.Lock()




×








NEW


225



        oldAaepMonitorData, _ := cont.getAaepEpgAttachDataLocked(aaepName, epgDn)




×







226



        cont.indexMutex.Unlock()




×







227






×








NEW


228



        cont.syncNADsWithAciState(aaepName, epgDn, oldAaepMonitorData, aaepMonitorData, "AaepEpgAttached")




×







229



}










230












231



func (cont *AciController) handleAaepEpgDetach(infraRsObjDn string) {




×







232



        aaepName := cont.matchesAEPFilter(infraRsObjDn)




×







233



        if aaepName == "" {




×







234



                cont.log.Debugf("Unable to find AAEP from %s in monitoring list", infraRsObjDn)




×







235



                return




×







236



        }




×







237












238



        epgDn := cont.getEpgDnFromInfraRsDn(infraRsObjDn)




×







239






×








NEW


240



        if !cont.isEpgAttachedWithAaep(epgDn) {




×








NEW


241



                cont.apicConn.UnsubscribeImmediateDnLocked(epgDn, []string{"tagAnnotation"})




×








NEW


242



        }




×







243












244



        cont.indexMutex.Lock()




×








NEW


245



        aaepMonitorData, _ := cont.getAaepEpgAttachDataLocked(aaepName, epgDn)




×







246



        cont.indexMutex.Unlock()




×







247






×







248



        if aaepMonitorData == nil || !cont.namespaceChecks(aaepMonitorData.namespaceName, epgDn) {




×








NEW


249



                cont.log.Debugf("Monitoring data not available for EPG %s with AAEP %s or namespace %s not found",




×








NEW


250



                        epgDn, aaepName, aaepMonitorData.namespaceName)




×







251



                return




×







252



        }




×







253












254



        cont.indexMutex.Lock()




×








NEW


255



        cont.removeAaepEpgAttachDataLocked(aaepName, epgDn)




×








NEW


256



        cont.deleteNetworkAttachmentDefinition(aaepName, aaepMonitorData, "AaepEpgDetached")




×








UNCOV

257



        cont.indexMutex.Unlock()




×







258



}










259












260



func (cont *AciController) handleAnnotationAdded(obj apicapi.ApicObject) bool {




×







261



        annotationDn := obj.GetDn()




×







262



        epgDn := annotationDn[:strings.Index(annotationDn, "/annotationKey-")]




×








NEW


263



        aaepMonitorDataMap := cont.getAaepMonitoringDataForEpg(epgDn)




×







264






×








NEW


265



        for aaepName, aaepMonitorData := range aaepMonitorDataMap {




×








NEW


266



                if aaepMonitorData == nil {




×








NEW


267



                        cont.log.Debugf("Insufficient data for NAD creation: Monitoring data not available for EPG %s with AAEP %s", epgDn, aaepName)




×








NEW


268



                        continue




×







269



                }










270













NEW


271



                if !cont.namespaceChecks(aaepMonitorData.namespaceName, epgDn) {




×








NEW


272



                        cont.log.Debugf("Insufficient data for NAD creation: Namespace not exist, in case of EPG %s with AAEP %s", epgDn, aaepName)




×








NEW


273



                        continue




×







274



                }










275













NEW


276



                cont.indexMutex.Lock()




×








NEW


277



                oldAaepMonitorData, _ := cont.getAaepEpgAttachDataLocked(aaepName, epgDn)




×








NEW


278



                cont.indexMutex.Unlock()




×








NEW


279



                cont.syncNADsWithAciState(aaepName, epgDn, oldAaepMonitorData, aaepMonitorData, "NamespaceAnnotationAdded")




×







280



        }










281












282



        return true




×







283



}










284












285



func (cont *AciController) handleAnnotationDeleted(annotationDn string) {




×







286



        epgDn := annotationDn[:strings.Index(annotationDn, "/annotationKey-")]




×







287






×








NEW


288



        aaepMonitorDataMap := cont.getAaepMonitoringDataForEpg(epgDn)




×







289






×








NEW


290



        for aaepName, aaepMonitorData := range aaepMonitorDataMap {




×







291



                cont.indexMutex.Lock()




×








NEW


292



                oldAaepMonitorData, _ := cont.getAaepEpgAttachDataLocked(aaepName, epgDn)




×







293



                cont.indexMutex.Unlock()




×







294






×








NEW


295



                if oldAaepMonitorData == nil {




×








NEW


296



                        cont.log.Debugf("Monitoring data not available for EPG %s with AAEP %s", epgDn, aaepName)




×








NEW


297



                        continue




×







298



                }










299













NEW


300



                if aaepMonitorData == nil {




×








NEW


301



                        cont.indexMutex.Lock()




×








NEW


302



                        cont.removeAaepEpgAttachDataLocked(aaepName, epgDn)




×








NEW


303



                        cont.deleteNetworkAttachmentDefinition(aaepName, oldAaepMonitorData, "NamespaceAnnotationRemoved")




×








NEW


304



                        cont.indexMutex.Unlock()




×








NEW


305



                        continue




×







306



                }










307













NEW


308



                cont.syncNADsWithAciState(aaepName, epgDn, oldAaepMonitorData, aaepMonitorData, "NamespaceAnnotationRemoved")




×







309



        }










310



}










311












312



func (cont *AciController) collectNadData(aaepEpgData *AaepEpgAttachData) *AaepMonitoringData {




×







313



        epgDn := aaepEpgData.epgDn




×







314



        epgAnnotations := cont.getEpgAnnotations(epgDn)




×







315



        namespaceName, nadName := cont.getSpecificEPGAnnotation(epgAnnotations)




×







316






×







317



        if !cont.namespaceChecks(namespaceName, epgDn) {




×







318



                return nil




×







319



        }




×







320












321



        aaepMonitoringData := &AaepMonitoringData{




×







322



                aaepEpgData:   *aaepEpgData,




×







323



                nadName:       nadName,




×







324



                namespaceName: namespaceName,




×







325



        }




×







326






×







327



        return aaepMonitoringData




×







328



}










329












330



func (cont *AciController) getAaepEpgAttachDataLocked(aaepName string, epgDn string) (*AaepMonitoringData, int) {




×







331



        aaepEpgDataList, exists := cont.sharedAaepMonitor[aaepName]




×







332



        if !exists || len(aaepEpgDataList) == 0 {




×








NEW


333



                cont.log.Debugf("AAEP %s EPG %s attachment data not found", aaepName, epgDn)




×







334



                return nil, -1




×







335



        }




×







336












337



        for dataIndex, aaepEpgData := range aaepEpgDataList {




×







338



                if aaepEpgData.aaepEpgData.epgDn == epgDn {




×








NEW


339



                        cont.log.Infof("Found attachment data: %v for EPG : %s AAEP: %s", aaepEpgData, epgDn, aaepName)




×







340



                        return aaepEpgData, dataIndex




×







341



                }




×







342



        }










343



        return nil, -1




×







344



}










345












346



func (cont *AciController) matchesAEPFilter(infraRsObjDn string) string {




×







347



        cont.indexMutex.Lock()




×







348



        defer cont.indexMutex.Unlock()




×







349



        var aaepName string




×







350



        for aaepName = range cont.sharedAaepMonitor {




×








NEW


351



                expectedPrefix := fmt.Sprintf("uni/infra/attentp-%s/", aaepName)




×







352



                if strings.HasPrefix(infraRsObjDn, expectedPrefix) {




×







353



                        return aaepName




×







354



                }




×







355



        }










356



        return ""




×







357



}










358












359



func (cont *AciController) getEpgDnFromInfraRsDn(infraRsObjDn string) string {




×







360



        re := regexp.MustCompile(`\[(.*?)\]`)




×







361



        match := re.FindStringSubmatch(infraRsObjDn)




×







362






×







363



        var epgDn string




×







364



        if len(match) > 1 {




×







365



                epgDn = match[1]




×







366



                return epgDn




×







367



        }




×







368












369



        return epgDn




×







370



}










371













NEW


372



func (cont *AciController) getAaepMonitoringDataForEpg(epgDn string) map[string]*AaepMonitoringData {




×








NEW


373



        aaepMonitorDataMap := make(map[string]*AaepMonitoringData)




×







374



        var aaepName string




×







375






×







376



        cont.indexMutex.Lock()




×







377



        defer cont.indexMutex.Unlock()




×







378



        for aaepName = range cont.sharedAaepMonitor {




×







379



                encap := cont.getEncapFromAaepEpgAttachObj(aaepName, epgDn)




×







380






×







381



                if encap != "" {




×







382



                        vlanID := cont.getVlanId(encap)




×







383



                        aaepEpgData := &AaepEpgAttachData{




×







384



                                epgDn:     epgDn,




×







385



                                encapVlan: vlanID,




×







386



                        }




×







387






×








NEW


388



                        aaepMonitorDataMap[aaepName] = cont.collectNadData(aaepEpgData)




×







389



                }




×







390



        }










391













NEW


392



        return aaepMonitorDataMap




×







393



}










394












395



func (cont *AciController) cleanAnnotationSubscriptions(aaepName string) {




×







396



        aaepEpgDataList := cont.getAaepEpgAttObjDetails(aaepName)




×







397



        if aaepEpgDataList == nil {




×







398



                return




×







399



        }




×







400












401



        for _, aaepEpgData := range aaepEpgDataList {




×







402



                cont.apicConn.UnsubscribeImmediateDnLocked(aaepEpgData.epgDn, []string{"tagAnnotation"})




×







403



        }




×







404



}










405












406



func (cont *AciController) syncNADsWithAciState(aaepName string, epgDn string, oldAaepMonitorData,










407



        aaepMonitorData *AaepMonitoringData, syncReason string) {




×







408



        if oldAaepMonitorData == nil {




×







409



                cont.indexMutex.Lock()




×








NEW


410



                needCacheChange := cont.createNetworkAttachmentDefinition(aaepName, aaepMonitorData, syncReason)




×







411



                if needCacheChange {




×







412



                        cont.sharedAaepMonitor[aaepName] = append(cont.sharedAaepMonitor[aaepName], aaepMonitorData)




×







413



                }




×







414



                cont.indexMutex.Unlock()




×







415



        } else {




×







416



                if oldAaepMonitorData.namespaceName != aaepMonitorData.namespaceName {




×







417



                        cont.indexMutex.Lock()




×








NEW


418



                        cont.removeAaepEpgAttachDataLocked(aaepName, epgDn)




×








NEW


419



                        cont.deleteNetworkAttachmentDefinition(aaepName, oldAaepMonitorData, syncReason)




×







420



                        cont.indexMutex.Unlock()




×







421






×







422



                        cont.indexMutex.Lock()




×








NEW


423



                        needCacheChange := cont.createNetworkAttachmentDefinition(aaepName, aaepMonitorData, syncReason)




×







424



                        if needCacheChange {




×







425



                                cont.sharedAaepMonitor[aaepName] = append(cont.sharedAaepMonitor[aaepName], aaepMonitorData)




×







426



                        }




×







427



                        cont.indexMutex.Unlock()




×







428



                        return




×







429



                }










430












431



                if oldAaepMonitorData.nadName != aaepMonitorData.nadName || oldAaepMonitorData.aaepEpgData.encapVlan != aaepMonitorData.aaepEpgData.encapVlan {




×







432



                        cont.indexMutex.Lock()




×








NEW


433



                        needCacheChange := cont.createNetworkAttachmentDefinition(aaepName, aaepMonitorData, syncReason)




×







434



                        if needCacheChange {




×








NEW


435



                                cont.removeAaepEpgAttachDataLocked(aaepName, epgDn)




×







436



                                cont.sharedAaepMonitor[aaepName] = append(cont.sharedAaepMonitor[aaepName], aaepMonitorData)




×







437



                        }




×







438



                        cont.indexMutex.Unlock()




×







439



                }










440



        }










441



}










442












443



func (cont *AciController) addDeferredNADs(namespaceName string) {




×







444



        cont.indexMutex.Lock()




×







445



        defer cont.indexMutex.Unlock()




×







446



        for aaepName := range cont.sharedAaepMonitor {




×







447



                aaepEpgDataList := cont.getAaepEpgAttObjDetails(aaepName)




×







448






×







449



                if aaepEpgDataList == nil {




×







450



                        continue




×







451



                }










452












453



                for _, aaepEpgData := range aaepEpgDataList {




×







454



                        aaepMonitoringData := cont.collectNadData(&aaepEpgData)




×







455



                        if aaepMonitoringData == nil || aaepMonitoringData.namespaceName != namespaceName {




×







456



                                continue




×







457



                        }










458













NEW


459



                        needCacheChange := cont.createNetworkAttachmentDefinition(aaepName, aaepMonitoringData, "NamespaceCreated")




×







460



                        if needCacheChange {




×







461



                                cont.sharedAaepMonitor[aaepName] = append(cont.sharedAaepMonitor[aaepName], aaepMonitoringData)




×







462



                        }




×







463



                }










464



        }










465



}










466












467



func (cont *AciController) cleanNADs(namespaceName string) {




×







468



        cont.indexMutex.Lock()




×







469



        defer cont.indexMutex.Unlock()




×







470



        for aaepName := range cont.sharedAaepMonitor {




×







471



                aaepEpgDataList, exists := cont.sharedAaepMonitor[aaepName]




×







472



                if !exists || len(aaepEpgDataList) == 0 {




×







473



                        continue




×







474



                }










475



                newAaepEpgDataList := []*AaepMonitoringData{}




×







476



                for _, aaepEpgData := range aaepEpgDataList {




×







477



                        if aaepEpgData.namespaceName != namespaceName {




×







478



                                newAaepEpgDataList = append(newAaepEpgDataList, aaepEpgData)




×







479



                        }




×







480



                }










481



                cont.sharedAaepMonitor[aaepName] = newAaepEpgDataList




×







482



        }










483



}










484












485



func (cont *AciController) getAaepEpgAttObjDetails(aaepName string) []AaepEpgAttachData {




×







486



        uri := fmt.Sprintf("/api/node/mo/uni/infra/attentp-%s.json?query-target=subtree&target-subtree-class=infraRsFuncToEpg", aaepName)




×







487






×







488



        resp, err := cont.apicConn.GetApicResponse(uri)




×







489



        if err != nil {




×







490



                cont.log.Errorf("Failed to get response from APIC: %v", err)




×







491



                return nil




×







492



        }




×







493












494



        if len(resp.Imdata) == 0 {




×








NEW


495



                cont.log.Debugf("Can't find EPGs attached with AAEP %s", aaepName)




×







496



                return nil




×







497



        }




×







498












499



        aaepEpgAttchDetails := make([]AaepEpgAttachData, 0)




×







500



        for _, respImdata := range resp.Imdata {




×







501



                aaepEpgAttachObj, ok := respImdata["infraRsFuncToEpg"]




×







502



                if !ok {




×








NEW


503



                        cont.log.Debugf("Empty AAEP EPG attachment object")




×







504



                        continue




×







505



                }










506












507



                if state, hasState := aaepEpgAttachObj.Attributes["state"].(string); hasState {




×







508



                        if state != "formed" {




×







509



                                aaepEpgAttchDn := aaepEpgAttachObj.Attributes["dn"].(string)




×








NEW


510



                                cont.log.Debugf("%s is with state: %s", aaepEpgAttchDn, state)




×







511



                                continue




×







512



                        }










513



                }










514



                vlanID := 0




×







515



                if encap, hasEncap := aaepEpgAttachObj.Attributes["encap"].(string); hasEncap {




×







516



                        vlanID = cont.getVlanId(encap)




×







517



                }




×







518












519



                aaepEpgData := AaepEpgAttachData{




×







520



                        epgDn:     aaepEpgAttachObj.Attributes["tDn"].(string),




×







521



                        encapVlan: vlanID,




×







522



                }




×







523



                aaepEpgAttchDetails = append(aaepEpgAttchDetails, aaepEpgData)




×







524



        }










525












526



        return aaepEpgAttchDetails




×







527



}










528












529



func (cont *AciController) getEpgAnnotations(epgDn string) map[string]string {




×







530



        uri := fmt.Sprintf("/api/node/mo/%s.json?query-target=subtree&target-subtree-class=tagAnnotation", epgDn)




×







531



        resp, err := cont.apicConn.GetApicResponse(uri)




×







532



        if err != nil {




×







533



                cont.log.Errorf("Failed to get response from APIC: %v", err)




×







534



                return nil




×







535



        }




×







536












537



        annotationsMap := make(map[string]string)




×







538



        for _, respImdata := range resp.Imdata {




×







539



                annotationObj, ok := respImdata["tagAnnotation"]




×







540



                if !ok {




×








NEW


541



                        cont.log.Debugf("Empty tag annotation of EPG %s", epgDn)




×







542



                        continue




×







543



                }










544












545



                key := annotationObj.Attributes["key"].(string)




×







546



                annotationsMap[key] = annotationObj.Attributes["value"].(string)




×







547



        }










548












549



        return annotationsMap




×







550



}










551












552



func (cont *AciController) getSpecificEPGAnnotation(annotations map[string]string) (string, string) {




×







553



        namespaceNameAnnotationKey := cont.config.CnoIdentifier + "-namespace"




×







554



        namespaceName, exists := annotations[namespaceNameAnnotationKey]




×







555



        if !exists {




×








NEW


556



                cont.log.Debugf("Annotation with key '%s' not found", namespaceNameAnnotationKey)




×







557



        }




×







558












559



        nadNameAnnotationKey := cont.config.CnoIdentifier + "-nad"




×







560



        nadName, exists := annotations[nadNameAnnotationKey]




×







561



        if !exists {




×








NEW


562



                cont.log.Debugf("Annotation with key '%s' not found", nadNameAnnotationKey)




×







563



        }




×







564



        return namespaceName, nadName




×







565



}










566












567



func (cont *AciController) namespaceChecks(namespaceName string, epgDn string) bool {




×







568



        if namespaceName == "" {




×







569



                cont.log.Debugf("Defering NAD operation for EPG %s: Namespace name not provided in EPG annotation", epgDn)




×







570



                return false




×







571



        }




×







572












573



        kubeClient := cont.env.(*K8sEnvironment).kubeClient




×







574



        _, err := kubeClient.CoreV1().Namespaces().Get(context.TODO(), namespaceName, metav1.GetOptions{})




×







575



        namespaceExists := err == nil




×







576



        if !namespaceExists {




×







577



                cont.log.Debugf("Defering NAD operation for EPG %s: Namespace %s not exists", epgDn, namespaceName)




×







578



                return false




×







579



        }




×







580












581



        return true




×







582



}










583












584



func (cont *AciController) reconcileNadData(aaepName string) {




×







585



        aaepEpgDataList := cont.getAaepEpgAttObjDetails(aaepName)




×







586






×







587



        for _, aaepEpgData := range aaepEpgDataList {




×







588



                aaepMonitoringData := cont.collectNadData(&aaepEpgData)




×







589



                if aaepMonitoringData == nil {




×







590



                        cont.apicConn.AddImmediateSubscriptionDnLocked(aaepEpgData.epgDn,




×







591



                                []string{"tagAnnotation"}, cont.handleAnnotationAdded,




×







592



                                cont.handleAnnotationDeleted)




×







593



                        continue




×







594



                }










595












596



                cont.indexMutex.Lock()




×








NEW


597



                needCacheChange := cont.createNetworkAttachmentDefinition(aaepName, aaepMonitoringData, "AaepAddedInCR")




×







598



                if needCacheChange {




×







599



                        cont.sharedAaepMonitor[aaepName] = append(cont.sharedAaepMonitor[aaepName], aaepMonitoringData)




×







600



                }




×







601



                cont.indexMutex.Unlock()




×







602






×







603



                cont.apicConn.AddImmediateSubscriptionDnLocked(aaepEpgData.epgDn,




×







604



                        []string{"tagAnnotation"}, cont.handleAnnotationAdded,




×







605



                        cont.handleAnnotationDeleted)




×







606



        }










607












608



        cont.indexMutex.Lock()




×







609



        if _, ok := cont.sharedAaepMonitor[aaepName]; !ok {




×







610



                cont.sharedAaepMonitor[aaepName] = []*AaepMonitoringData{}




×







611



        }




×







612



        cont.indexMutex.Unlock()




×







613



}










614












615



// clean converts a string to lowercase, removes underscores and dots,










616



// and replaces any other invalid character with a hyphen.











NEW


617



func cleanApicResourceNames(apicResource string) string {




×








NEW


618



        apicResource = strings.ToLower(apicResource)




×








NEW


619



        var stringBuilder strings.Builder




×








NEW


620



        for _, character := range apicResource {




×








NEW


621



                switch {




×








NEW


622



                case character == '_' || character == '.':




×








NEW


623



                        continue




×








NEW


624



                case (character >= 'a' && character <= 'z') || (character >= '0' && character <= '9') || character == '-':




×








NEW


625



                        stringBuilder.WriteRune(character)




×








NEW


626



                default:




×








NEW


627



                        stringBuilder.WriteRune('-')




×







628



                }










629



        }











NEW


630



        return strings.Trim(stringBuilder.String(), "-")




×







631



}










632













NEW


633



func (cont *AciController) generateDefaultNadName(aaepName, epgDn string) string {




×







634



        parts := strings.Split(epgDn, "/")




×







635






×







636



        tenant := parts[1][3:]




×







637



        appProfile := parts[2][3:]




×







638



        epgName := parts[3][4:]




×







639






×








NEW


640



        apicResourceNames := tenant + appProfile + epgName + aaepName




×








NEW


641



        hashBytes := sha256.Sum256([]byte(apicResourceNames))




×








NEW


642



        hash := hex.EncodeToString(hashBytes[:])[:16]




×








NEW


643






×








NEW


644



        return fmt.Sprintf("%s-%s-%s-%s",




×








NEW


645



                cleanApicResourceNames(tenant), cleanApicResourceNames(appProfile), cleanApicResourceNames(epgName), hash)




×








UNCOV

646



}




×







647













NEW


648



func (cont *AciController) isNADUpdateRequired(aaepName string, nadData *AaepMonitoringData, existingNAD *nadapi.NetworkAttachmentDefinition) bool {




×







649



        vlanID := nadData.aaepEpgData.encapVlan




×







650



        namespaceName := nadData.namespaceName




×







651



        customNadName := nadData.nadName




×








NEW


652



        defaultNadName := cont.generateDefaultNadName(aaepName, nadData.aaepEpgData.epgDn)




×







653



        existingAnnotaions := existingNAD.ObjectMeta.Annotations




×







654



        if existingAnnotaions != nil {




×







655



                if existingNAD.ObjectMeta.Annotations["aci-sync-status"] == "out-of-sync" || existingNAD.ObjectMeta.Annotations["cno-name"] != customNadName {




×







656



                        return true




×







657



                }




×







658



        } else {




×







659



                // NAD exists, check if VLAN needs to be updated




×







660



                existingConfig := existingNAD.Spec.Config




×







661



                if existingConfig != "" {




×







662



                        var existingCNVConfig map[string]interface{}




×







663



                        if json.Unmarshal([]byte(existingConfig), &existingCNVConfig) == nil {




×







664



                                if existingVLAN, ok := existingCNVConfig["vlan"].(float64); ok {




×







665



                                        if int(existingVLAN) == vlanID {




×







666



                                                // VLAN hasn't changed, no update needed




×







667



                                                cont.log.Infof("NetworkAttachmentDefinition %s already exists with correct VLAN %d in namespace %s", defaultNadName, vlanID, namespaceName)




×







668



                                                return false




×







669



                                        }




×







670



                                } else if vlanID == 0 {




×







671



                                        // Both existing and new have no VLAN, no update needed




×







672



                                        cont.log.Infof("NetworkAttachmentDefinition %s already exists with no VLAN in namespace %s", defaultNadName, namespaceName)




×







673



                                        return false




×







674



                                }




×







675



                        }










676



                }










677



        }










678












679



        return true




×







680



}










681













NEW


682



func (cont *AciController) createNetworkAttachmentDefinition(aaepName string, nadData *AaepMonitoringData, createReason string) bool {




×







683



        bridge := cont.config.BridgeName




×







684



        if bridge == "" {




×







685



                cont.log.Errorf("Linux bridge name must be specified when creating NetworkAttachmentDefinitions")




×







686



                return false




×







687



        }




×







688












689



        vlanID := nadData.aaepEpgData.encapVlan




×







690



        namespaceName := nadData.namespaceName




×







691



        customNadName := nadData.nadName




×








NEW


692



        defaultNadName := cont.generateDefaultNadName(aaepName, nadData.aaepEpgData.epgDn)




×







693



        nadClient := cont.env.(*K8sEnvironment).nadClient




×







694



        mtu := 1500




×







695






×







696



        // Check if NAD already exists




×







697



        existingNAD, err := nadClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespaceName).Get(context.TODO(), defaultNadName, metav1.GetOptions{})




×







698



        nadExists := err == nil




×







699






×








NEW


700



        if nadExists && !cont.isNADUpdateRequired(aaepName, nadData, existingNAD) {




×







701



                return true




×







702



        }




×







703












704



        cnvBridgeConfig := map[string]any{




×







705



                "cniVersion":       "0.3.1",




×







706



                "name":             defaultNadName,




×







707



                "type":             "bridge",




×







708



                "isDefaultGateway": true,




×







709



                "bridge":           bridge,




×







710



                "mtu":              mtu,




×







711



        }




×







712






×







713



        if vlanID > 0 {




×







714



                cnvBridgeConfig["vlan"] = vlanID




×







715



        }




×







716












717



        configJSON, err := json.Marshal(cnvBridgeConfig)




×







718



        if err != nil {




×







719



                cont.log.Errorf("Failed to marshal CNV bridge config: %v", err)




×







720



                return false




×







721



        }




×







722












723



        nad := &nadapi.NetworkAttachmentDefinition{




×







724



                TypeMeta: metav1.TypeMeta{




×







725



                        APIVersion: "k8s.cni.cncf.io/v1",




×







726



                        Kind:       "NetworkAttachmentDefinition",




×







727



                },




×







728



                ObjectMeta: metav1.ObjectMeta{




×







729



                        Name:      defaultNadName,




×







730



                        Namespace: namespaceName,




×







731



                        Labels: map[string]string{




×







732



                                "managed-by": "cisco-network-operator",




×







733



                                "vlan":       strconv.Itoa(vlanID),




×







734



                        },




×







735



                        Annotations: map[string]string{




×







736



                                "managed-by":      "cisco-network-operator",




×







737



                                "vlan":            strconv.Itoa(vlanID),




×







738



                                "cno-name":        customNadName,




×







739



                                "aci-sync-status": "in-sync",




×








NEW


740



                                "aaep-name":       aaepName,




×








NEW


741



                                "epg-dn":          nadData.aaepEpgData.epgDn,




×







742



                        },




×







743



                },




×







744



                Spec: nadapi.NetworkAttachmentDefinitionSpec{




×







745



                        Config: string(configJSON),




×







746



                },




×







747



        }




×







748






×







749



        if nadExists {




×







750



                nad.ObjectMeta.ResourceVersion = existingNAD.ObjectMeta.ResourceVersion




×







751






×







752



                updatedNad, err := nadClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespaceName).Update(context.TODO(), nad, metav1.UpdateOptions{})




×







753






×







754



                if err != nil {




×







755



                        cont.log.Errorf("Failed to update NetworkAttachmentDefinition %s from namespace %s : %v", customNadName, namespaceName, err)




×







756



                        return false




×







757



                }




×







758












759



                cont.log.Debugf("Existing NAD Annotations: %v, %s", existingNAD.ObjectMeta.Annotations, createReason)




×







760



                if existingNAD.ObjectMeta.Annotations["aci-sync-status"] == "out-of-sync" {




×







761



                        cont.submitEvent(updatedNad, createReason, cont.getNADRevampMessage(createReason))




×







762



                }




×







763



                cont.log.Infof("Updated NetworkAttachmentDefinition %s from namespace %s", defaultNadName, namespaceName)




×







764



        } else {




×







765



                _, err = nadClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespaceName).Create(context.TODO(), nad, metav1.CreateOptions{})




×







766



                if err != nil {




×







767



                        cont.log.Errorf("Failed to create NetworkAttachmentDefinition %s in namespace %s : %v", customNadName, namespaceName, err)




×







768



                        return false




×







769



                }




×







770



                cont.log.Infof("Created NetworkAttachmentDefinition %s in namespace %s", defaultNadName, namespaceName)




×







771



        }










772












773



        return true




×







774



}










775













NEW


776



func (cont *AciController) deleteNetworkAttachmentDefinition(aaepName string, nadData *AaepMonitoringData, deleteReason string) {




×







777



        namespaceName := nadData.namespaceName




×







778



        epgDn := nadData.aaepEpgData.epgDn




×







779






×







780



        kubeClient := cont.env.(*K8sEnvironment).kubeClient




×







781



        _, err := kubeClient.CoreV1().Namespaces().Get(context.TODO(), namespaceName, metav1.GetOptions{})




×







782



        namespaceExists := err == nil




×







783



        if !namespaceExists {




×







784



                cont.log.Debugf("Defering NAD deletion for EPG %s: Namespace %s not exists", epgDn, namespaceName)




×







785



                return




×







786



        }




×







787













NEW


788



        nadName := cont.generateDefaultNadName(aaepName, nadData.aaepEpgData.epgDn)




×







789



        nadClient := cont.env.(*K8sEnvironment).nadClient




×







790






×







791



        nadDetails, err := nadClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespaceName).Get(context.TODO(), nadName, metav1.GetOptions{})




×







792



        nadExists := err == nil




×







793






×







794



        if nadExists {




×







795



                if !cont.isVmmLiteNAD(nadDetails) {




×







796



                        return




×







797



                }




×







798












799



                if cont.isNADinUse(namespaceName, nadName) {




×







800



                        nadDetails.ObjectMeta.Annotations["aci-sync-status"] = "out-of-sync"




×







801



                        _, err = nadClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespaceName).Update(context.TODO(), nadDetails, metav1.UpdateOptions{})




×







802



                        if err != nil {




×







803



                                cont.log.Errorf("Failed to add out-of-sync annotation to the NAD %s from namespace %s : %v", nadName, namespaceName, err)




×







804



                                return




×







805



                        }




×







806



                        cont.submitEvent(nadDetails, deleteReason, cont.getNADDeleteMessage(deleteReason))




×







807



                        cont.log.Infof("Added annotation out-of-sync for NAD %s from namespace %s", nadName, namespaceName)




×







808



                        return




×







809



                }










810












811



                delete(nadDetails.ObjectMeta.Annotations, "managed-by")




×







812



                _, err = nadClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespaceName).Update(context.TODO(), nadDetails, metav1.UpdateOptions{})




×







813



                if err != nil {




×







814



                        cont.log.Errorf("Failed to remove VMM lite annotation from NetworkAttachmentDefinition %s from namespace %s: %v", nadName, namespaceName, err)




×







815



                        return




×







816



                }




×







817












818



                nadClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(namespaceName).Delete(context.TODO(), nadName, metav1.DeleteOptions{})




×







819



                cont.log.Infof("Deleted NAD %s from %s namespace", nadName, namespaceName)




×







820



        } else {




×







821



                cont.log.Debugf("NAD %s not there to delete in namespace %s", nadName, namespaceName)




×







822



        }




×







823



}










824












825



func (cont *AciController) getVlanId(encap string) int {




×







826



        if after, ok := strings.CutPrefix(encap, "vlan-"); ok {




×







827



                vlanStr := after




×







828



                if vlanID, err := strconv.Atoi(vlanStr); err == nil && vlanID > 0 {




×







829



                        return vlanID




×







830



                }




×







831



        } else if after, ok := strings.CutPrefix(encap, "vlan"); ok {




×







832



                vlanStr := after




×







833



                if vlanID, err := strconv.Atoi(vlanStr); err == nil && vlanID > 0 {




×







834



                        return vlanID




×







835



                }




×







836



        }










837












838



        return 0




×







839



}










840












841



func (cont *AciController) getAaepDiff(crAaeps []string) (addedAaeps, removedAaeps []string) {




×







842



        crAaepMap := make(map[string]bool)




×







843



        for _, crAaep := range crAaeps {




×







844



                crAaepMap[crAaep] = true




×







845



        }




×







846












847



        cont.indexMutex.Lock()




×







848



        for _, crAaep := range crAaeps {




×







849



                if _, ok := cont.sharedAaepMonitor[crAaep]; !ok {




×







850



                        addedAaeps = append(addedAaeps, crAaep)




×







851



                }




×







852



        }










853



        cont.indexMutex.Unlock()




×







854






×







855



        cont.indexMutex.Lock()




×







856



        for cachedAaep := range cont.sharedAaepMonitor {




×







857



                if !crAaepMap[cachedAaep] {




×







858



                        removedAaeps = append(removedAaeps, cachedAaep)




×







859



                }




×







860



        }










861



        cont.indexMutex.Unlock()




×







862






×







863



        return




×







864



}










865












866



func (cont *AciController) getEncapFromAaepEpgAttachObj(aaepName, epgDn string) string {




×







867



        uri := fmt.Sprintf("/api/node/mo/uni/infra/attentp-%s/gen-default/rsfuncToEpg-[%s].json?query-target=self", aaepName, epgDn)




×







868



        resp, err := cont.apicConn.GetApicResponse(uri)




×







869



        if err != nil {




×







870



                cont.log.Errorf("Failed to get response from APIC: AAEP %s and EPG %s ERROR: %v", aaepName, epgDn, err)




×







871



                return ""




×







872



        }




×







873












874



        for _, obj := range resp.Imdata {




×







875



                lresp, ok := obj["infraRsFuncToEpg"]




×







876



                if !ok {




×







877



                        cont.log.Errorf("InfraRsFuncToEpg object not found in response for %s", uri)




×







878



                        break




×







879



                }










880



                if val, ok := lresp.Attributes["encap"]; ok {




×







881



                        encap := val.(string)




×







882



                        return encap




×







883



                } else {




×







884



                        cont.log.Errorf("Encap missing for infraRsFuncToEpg object for %s: %v", uri, err)




×







885



                        break




×







886



                }










887



        }










888












889



        return ""




×







890



}










891












892



func (cont *AciController) isVmmLiteNAD(nadDetails *nadapi.NetworkAttachmentDefinition) bool {




×







893



        return nadDetails.ObjectMeta.Annotations["managed-by"] == "cisco-network-operator"




×







894



}




×







895












896



func (cont *AciController) isNADinUse(namespaceName string, nadName string) bool {




×







897



        kubeClient := cont.env.(*K8sEnvironment).kubeClient




×







898



        pods, err := kubeClient.CoreV1().Pods(namespaceName).List(context.TODO(), metav1.ListOptions{})




×







899



        if err == nil {




×







900



                var networks []map[string]string




×







901



                for _, pod := range pods.Items {




×







902



                        networksAnn, ok := pod.Annotations["k8s.v1.cni.cncf.io/networks"]




×







903



                        if ok && (networksAnn == nadName) {




×







904



                                cont.log.Infof("NAD %s is still used by Pod %s/%s", nadName, namespaceName, pod.Name)




×







905



                                return true




×







906



                        }




×







907



                        if err := json.Unmarshal([]byte(networksAnn), &networks); err != nil {




×







908



                                cont.log.Errorf("Error while getting pod annotations: %v", err)




×







909



                                return false




×







910



                        }




×







911



                        for _, network := range networks {




×







912



                                if ok && (network["name"] == nadName) {




×







913



                                        cont.log.Infof("NAD %s is still used by VM %s/%s", nadName, namespaceName, pod.Name)




×







914



                                        return true




×







915



                                }




×







916



                        }










917



                }










918



        }










919



        return false




×







920



}










921












922



func (cont *AciController) getNADDeleteMessage(deleteReason string) string {




×







923



        messagePrefix := "NAD is in use by pods: "




×







924



        switch {




×







925



        case deleteReason == "NamespaceAnnotationRemoved":




×








NEW


926



                return messagePrefix + "Either EPG deleted or namespace name EPG annotaion removed"




×







927



        case deleteReason == "AaepEpgDetached":




×







928



                return messagePrefix + "EPG detached from AAEP"




×







929



        case deleteReason == "CRDeleted":




×







930



                return messagePrefix + "aaepmonitor CR deleted"




×







931



        case deleteReason == "AaepRemovedFromCR":




×







932



                return messagePrefix + "AAEP removed from aaepmonitor CR"




×







933



        }










934



        return messagePrefix + "One or many pods are using NAD"




×







935



}










936












937



func (cont *AciController) getNADRevampMessage(createReason string) string {




×







938



        messagePrefix := "NAD is in sync: "




×







939



        switch {




×







940



        case createReason == "NamespaceAnnotationAdded":




×







941



                return messagePrefix + "Namespace name EPG annotaion added"




×







942



        case createReason == "AaepEpgAttached":




×







943



                return messagePrefix + "EPG attached with AAEP"




×







944



        case createReason == "AaepAddedInCR":




×







945



                return messagePrefix + "AAEP added back in aaepmonitor CR"




×







946



        case createReason == "NamespaceCreated":




×







947



                return messagePrefix + "Namespace created back"




×







948



        }










949



        return messagePrefix + "NAD synced with ACI"




×







950



}










951













NEW


952



func (cont *AciController) isEpgAttachedWithAaep(epgDn string) bool {




×








NEW


953



        for aaepName := range cont.sharedAaepMonitor {




×








NEW


954



                encap := cont.getEncapFromAaepEpgAttachObj(aaepName, epgDn)




×








NEW


955



                if encap != "" {




×








NEW


956



                        return true




×








NEW


957



                }




×







958



        }











NEW


959



        return false




×







960



}










961













NEW


962



func (cont *AciController) removeAaepEpgAttachDataLocked(aaepName, epgDn string) {




×








NEW


963



        aaepEpgAttachDataList, ok := cont.sharedAaepMonitor[aaepName]




×








NEW


964



        if !ok {




×








NEW


965



                return




×








NEW


966



        }




×








NEW


967



        for dataIndex, aaepEpgAttachData := range aaepEpgAttachDataList {




×








NEW


968



                if aaepEpgAttachData.aaepEpgData.epgDn == epgDn {




×








NEW


969



                        cont.sharedAaepMonitor[aaepName] = append(aaepEpgAttachDataList[:dataIndex], aaepEpgAttachDataList[dataIndex+1:]...)




×








NEW


970



                        return




×








NEW


971



                }




×







972



        }










973



}
























    

  • 

    •