18127951005

Committed 30 Sep 2025 11:12AM UTC coverage: 72.967% (-0.3%) from 73.233%

Build # 18127951005

Build Type

Pull #38

github

Committed by

web-flow

Commit Message

Merge 32c7ad5bd into 18355eef5

Pull Request Pull Request #38: feat(entity): add single group filter and docs

Run Details

18 of 28 new or added lines in 1 file covered. (64.29%)

9 existing lines in 1 file now uncovered.

359 of 492 relevant lines covered (72.97%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

65.93

/cortex/table_cortex_entity.go

package cortex

import (
        "context"
        "fmt"
        "strconv"
        "strings"

        "github.com/imroc/req/v3"
        "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
        "github.com/turbot/steampipe-plugin-sdk/v5/plugin"
        "github.com/turbot/steampipe-plugin-sdk/v5/plugin/quals"
        "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform"
)

type ScalarOrMap struct {
        Scalar interface{}
        Map    map[string]interface{}
}

func (s *ScalarOrMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
        if err := unmarshal(&s.Map); err == nil {
                return nil
        }
        if err := unmarshal(&s.Scalar); err != nil {
                return err
        }
        return nil
}

func (s *ScalarOrMap) Value() interface{} {
        if s.Scalar != nil {
                return s.Scalar
        }
        return s.Map
}

type CortexEntityResponse struct {
        Entities   []CortexEntityElement `yaml:"entities"`
        Page       int                   `yaml:"page"`
        TotalPages int                   `yaml:"totalPages"`
        Total      int                   `yaml:"total"`
}

type CortexEntityElement struct {
        Name        string                        `yaml:"name"`
        Tag         string                        `yaml:"tag"`
        Description string                        `yaml:"description"`
        Type        string                        `yaml:"type"`
        Hierarchy   CortexEntityElementHierarchy  `yaml:"hierarchy"`
        Groups      []string                      `yaml:"groups"`
        Metadata    []CortexEntityElementMetadata `yaml:"metadata"`
        LastUpdated string                        `yaml:"lastUpdated"`
        Links       []CortexLink                  `yaml:"links"`
        Archived    bool                          `yaml:"isArchived"`
        Git         CortexGithub                  `yaml:"git"`
        Slack       []CortexSlackChannel          `yaml:"slackChannels"`
        Owners      CortexEntityOwners            `yaml:"owners"`
}

type CortexEntityElementHierarchy struct {
        Parents []CortexTag `yaml:"parents"`
}

type CortexEntityElementMetadata struct {
        Key   string      `yaml:"key"`
        Value ScalarOrMap `yaml:"value"`
}

type CortexEntityOwners struct {
        Teams       []CortexEntityOwnersTeam       `yaml:"teams"`
        Individuals []CortexEntityOwnersIndividual `yaml:"individuals"`
}

type CortexEntityOwnersTeam struct {
        Tag string `yaml:"tag"`
}

type CortexEntityOwnersIndividual struct {
        Email string `yaml:"email"`
}

func tableCortexEntity() *plugin.Table {
        return &plugin.Table{
                Name:        "cortex_entity",
                Description: "Cortex list entities api.",
                List: &plugin.ListConfig{
                        Hydrate: listEntitiesHydrator,
                        KeyColumns: []*plugin.KeyColumn{
                                {Name: "archived", Require: plugin.Optional},
                                {Name: "type", Require: plugin.Optional},
                                {Name: "groups", Require: plugin.Optional, Operators: []string{"=", "?", "?|"}},
                        },
                },
                Columns: []*plugin.Column{
                        {Name: "name", Type: proto.ColumnType_STRING, Description: "Pretty name of the entity."},
                        {Name: "tag", Type: proto.ColumnType_STRING, Description: "The x-cortex-tag of the entity."},
                        {Name: "description", Type: proto.ColumnType_STRING, Description: "Description."},
                        {Name: "type", Type: proto.ColumnType_STRING, Description: "Entity Type."},
                        {Name: "parents", Type: proto.ColumnType_JSON, Description: "Parents of the entity.", Transform: FromStructSlice[CortexTag]("Hierarchy.Parents", "Tag")},
                        {Name: "groups", Type: proto.ColumnType_JSON, Description: "Groups, kind of like tags."},
                        {Name: "metadata", Type: proto.ColumnType_JSON, Description: "Raw custom metadata", Transform: transform.FromField("Metadata").Transform(TagArrayToMap)},
                        {Name: "last_updated", Type: proto.ColumnType_TIMESTAMP, Description: "Last updated time."},
                        {Name: "links", Type: proto.ColumnType_JSON, Description: "List of links", Transform: FromStructSlice[CortexLink]("Links", "Url")},
                        {Name: "archived", Type: proto.ColumnType_BOOL, Description: "Is archived."},
                        {Name: "repository", Type: proto.ColumnType_STRING, Description: "Git repo full name", Transform: transform.FromField("Git.Repository")},
                        {Name: "slack_channels", Type: proto.ColumnType_JSON, Description: "List of string slack channels"},
                        {Name: "owner_teams", Type: proto.ColumnType_JSON, Description: "List of owning team tags", Transform: FromStructSlice[CortexEntityOwnersTeam]("Owners.Teams", "Tag")},
                        {Name: "owner_individuals", Type: proto.ColumnType_JSON, Description: "List of owning individuals emails", Transform: FromStructSlice[CortexEntityOwnersIndividual]("Owners.Individuals", "Email")},
                },
        }
}

func listEntitiesHydrator(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
        logger := plugin.Logger(ctx)
        config := GetConfig(d.Connection)
        client := CortexHTTPClient(ctx, config)
        hydratorWriter := QueryDataWriter{d}

        // Extract parameters from QueryData
        archived := "false"
        if d.EqualsQuals["archived"] != nil && d.EqualsQuals["archived"].GetBoolValue() {
                logger.Debug("listEntitiesHydrator", "archived", d.EqualsQuals["archived"])
                archived = "true"
        }
        types := ""
        if d.EqualsQuals["type"] != nil {
                // When doing a "where in ()" steampipe does multiple separate calls to listEntities
                types = d.EqualsQuals["type"].GetStringValue()
        }
        groups := ""
        logger.Debug("listEntitiesHydrator", "quals", d.Quals)
        if d.Quals["groups"] != nil {
                groupFilters := buildGroupFilters(d.Quals["groups"].Quals)
                logger.Debug("listEntitiesHydrator", "groupFilters", groupFilters)
                if len(groupFilters) > 0 {
                        groups = strings.Join(groupFilters, ",")
                }
        }
        logger.Info("listEntitiesHydrator", "archived", archived, "types", types, "groups", groups)
        return nil, listEntities(ctx, client, &hydratorWriter, archived, types, groups)
}

func listEntities(ctx context.Context, client *req.Client, writer HydratorWriter, archived string, types string, groups string) error {
        logger := plugin.Logger(ctx)

        var response CortexEntityResponse
        var page int = 0
        for {
                logger.Debug("listEntities", "page", page)
                resp := client.
                        Get("/api/v1/catalog").
                        // Filters
                        SetQueryParam("includeArchived", archived).
                        SetQueryParam("types", types).
                        SetQueryParam("groups", groups).
                        // Options
                        SetQueryParam("yaml", "false").
                        SetQueryParam("includeMetadata", "true").
                        SetQueryParam("includeLinks", "true").
                        SetQueryParam("includeSlackChannels", "true").
                        SetQueryParam("includeOwners", "true").
                        SetQueryParam("includeHierarchyFields", "true").
                        // Pagination
                        SetQueryParam("pageSize", "1000").
                        SetQueryParam("page", strconv.Itoa(page)).
                        Do(ctx)

                // Check for HTTP errors
                if resp.IsErrorState() {
                        logger.Error("listEntities", "Status", resp.Status, "Body", resp.String())
                        return fmt.Errorf("error from cortex API %s: %s", resp.Status, resp.String())
                }

                // Unmarshal the response and check for unmarshal errors
                err := resp.Into(&response)
                if err != nil {
                        logger.Error("listEntities", "page", page, "Error", err)
                        return err
                }

                logger.Debug("listEntities", "totalPages", response.TotalPages, "total", response.Total)

                for _, result := range response.Entities {
                        // send the item to steampipe
                        writer.StreamListItem(ctx, result)
                        // Context can be cancelled due to manual cancellation or the limit has been hit
                        if writer.RowsRemaining(ctx) == 0 {
                                logger.Debug("listEntities", "RowsRemaining", writer.RowsRemaining(ctx))
                                return nil
                        }
                }
                page++
                if page >= response.TotalPages {
                        logger.Debug("listEntities", "page", page, "totalPages", response.TotalPages)
                        break
                }
        }
        return nil
}

func buildGroupFilters(groupQuals []*quals.Qual) []string {
        var groupFilters []string
        for _, q := range groupQuals {
                switch q.Operator {
                case quals.QualOperatorJsonbExistsOne, quals.QualOperatorEqual:
                        if value := q.Value.GetStringValue(); value != "" {
                                groupFilters = append(groupFilters, value)
                        }
                case quals.QualOperatorJsonbExistsAny:
                        if listValue := q.Value.GetListValue(); listValue != nil {
                                for _, v := range listValue.Values {
                                        if value := v.GetStringValue(); value != "" {
                                                groupFilters = append(groupFilters, value)
                                        }
                                }
                        }
                }
        }
        return groupFilters
}

1	package cortex
2
3	import (
4	"context"
5	"fmt"
6	"strconv"
7	"strings"
8
9	"github.com/imroc/req/v3"
10	"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
11	"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
12	"github.com/turbot/steampipe-plugin-sdk/v5/plugin/quals"
13	"github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform"
14	)
15
16	type ScalarOrMap struct {
17	Scalar interface{}
18	Map map[string]interface{}
19	}
20
21	func (s *ScalarOrMap) UnmarshalYAML(unmarshal func(interface{}) error) error {	×
22	if err := unmarshal(&s.Map); err == nil {	×
23	return nil	×
24	}	×
25	if err := unmarshal(&s.Scalar); err != nil {	×
26	return err	×
27	}	×
28	return nil	×
29	}
30
31	func (s *ScalarOrMap) Value() interface{} {	×
32	if s.Scalar != nil {	×
33	return s.Scalar	×
34	}	×
35	return s.Map	×
36	}
37
38	type CortexEntityResponse struct {
39	Entities []CortexEntityElement `yaml:"entities"`
40	Page int `yaml:"page"`
41	TotalPages int `yaml:"totalPages"`
42	Total int `yaml:"total"`
43	}
44
45	type CortexEntityElement struct {
46	Name string `yaml:"name"`
47	Tag string `yaml:"tag"`
48	Description string `yaml:"description"`
49	Type string `yaml:"type"`
50	Hierarchy CortexEntityElementHierarchy `yaml:"hierarchy"`
51	Groups []string `yaml:"groups"`
52	Metadata []CortexEntityElementMetadata `yaml:"metadata"`
53	LastUpdated string `yaml:"lastUpdated"`
54	Links []CortexLink `yaml:"links"`
55	Archived bool `yaml:"isArchived"`
56	Git CortexGithub `yaml:"git"`
57	Slack []CortexSlackChannel `yaml:"slackChannels"`
58	Owners CortexEntityOwners `yaml:"owners"`
59	}
60
61	type CortexEntityElementHierarchy struct {
62	Parents []CortexTag `yaml:"parents"`
63	}
64
65	type CortexEntityElementMetadata struct {
66	Key string `yaml:"key"`
67	Value ScalarOrMap `yaml:"value"`
68	}
69
70	type CortexEntityOwners struct {
71	Teams []CortexEntityOwnersTeam `yaml:"teams"`
72	Individuals []CortexEntityOwnersIndividual `yaml:"individuals"`
73	}
74
75	type CortexEntityOwnersTeam struct {
76	Tag string `yaml:"tag"`
77	}
78
79	type CortexEntityOwnersIndividual struct {
80	Email string `yaml:"email"`
81	}
82
83	func tableCortexEntity() *plugin.Table {	1✔
84	return &plugin.Table{	1✔
85	Name: "cortex_entity",	1✔
86	Description: "Cortex list entities api.",	1✔
87	List: &plugin.ListConfig{	1✔
88	Hydrate: listEntitiesHydrator,	1✔
89	KeyColumns: []*plugin.KeyColumn{	1✔
90	{Name: "archived", Require: plugin.Optional},	1✔
91	{Name: "type", Require: plugin.Optional},	1✔
92	{Name: "groups", Require: plugin.Optional, Operators: []string{"=", "?", "?\|"}},	1✔
93	},	1✔
94	},	1✔
95	Columns: []*plugin.Column{	1✔
96	{Name: "name", Type: proto.ColumnType_STRING, Description: "Pretty name of the entity."},	1✔
97	{Name: "tag", Type: proto.ColumnType_STRING, Description: "The x-cortex-tag of the entity."},	1✔
98	{Name: "description", Type: proto.ColumnType_STRING, Description: "Description."},	1✔
99	{Name: "type", Type: proto.ColumnType_STRING, Description: "Entity Type."},	1✔
100	{Name: "parents", Type: proto.ColumnType_JSON, Description: "Parents of the entity.", Transform: FromStructSlice[CortexTag]("Hierarchy.Parents", "Tag")},	1✔
101	{Name: "groups", Type: proto.ColumnType_JSON, Description: "Groups, kind of like tags."},	1✔
102	{Name: "metadata", Type: proto.ColumnType_JSON, Description: "Raw custom metadata", Transform: transform.FromField("Metadata").Transform(TagArrayToMap)},	1✔
103	{Name: "last_updated", Type: proto.ColumnType_TIMESTAMP, Description: "Last updated time."},	1✔
104	{Name: "links", Type: proto.ColumnType_JSON, Description: "List of links", Transform: FromStructSlice[CortexLink]("Links", "Url")},	1✔
105	{Name: "archived", Type: proto.ColumnType_BOOL, Description: "Is archived."},	1✔
106	{Name: "repository", Type: proto.ColumnType_STRING, Description: "Git repo full name", Transform: transform.FromField("Git.Repository")},	1✔
107	{Name: "slack_channels", Type: proto.ColumnType_JSON, Description: "List of string slack channels"},	1✔
108	{Name: "owner_teams", Type: proto.ColumnType_JSON, Description: "List of owning team tags", Transform: FromStructSlice[CortexEntityOwnersTeam]("Owners.Teams", "Tag")},	1✔
109	{Name: "owner_individuals", Type: proto.ColumnType_JSON, Description: "List of owning individuals emails", Transform: FromStructSlice[CortexEntityOwnersIndividual]("Owners.Individuals", "Email")},	1✔
110	},	1✔
111	}	1✔
112	}	1✔
113
114	func listEntitiesHydrator(ctx context.Context, d plugin.QueryData, h plugin.HydrateData) (interface{}, error) {	×
115	logger := plugin.Logger(ctx)	×
116	config := GetConfig(d.Connection)	×
117	client := CortexHTTPClient(ctx, config)	×
118	hydratorWriter := QueryDataWriter{d}	×
119		×
120	// Extract parameters from QueryData	×
121	archived := "false"	×
122	if d.EqualsQuals["archived"] != nil && d.EqualsQuals["archived"].GetBoolValue() {	×
123	logger.Debug("listEntitiesHydrator", "archived", d.EqualsQuals["archived"])	×
124	archived = "true"	×
125	}	×
126	types := ""	×
127	if d.EqualsQuals["type"] != nil {	×
128	// When doing a "where in ()" steampipe does multiple separate calls to listEntities	×
129	types = d.EqualsQuals["type"].GetStringValue()	×
130	}	×
NEW 131	groups := ""	×
NEW 132	logger.Debug("listEntitiesHydrator", "quals", d.Quals)	×
NEW 133	if d.Quals["groups"] != nil {	×
NEW 134	groupFilters := buildGroupFilters(d.Quals["groups"].Quals)	×
NEW UNCOV 135	logger.Debug("listEntitiesHydrator", "groupFilters", groupFilters)	×
NEW UNCOV 136	if len(groupFilters) > 0 {	×
NEW 137	groups = strings.Join(groupFilters, ",")	×
NEW 138	}	×
139	}
NEW UNCOV 140	logger.Info("listEntitiesHydrator", "archived", archived, "types", types, "groups", groups)	×
NEW 141	return nil, listEntities(ctx, client, &hydratorWriter, archived, types, groups)	×
142	}
143
144	func listEntities(ctx context.Context, client *req.Client, writer HydratorWriter, archived string, types string, groups string) error {	1✔
145	logger := plugin.Logger(ctx)	1✔
146		1✔
147	var response CortexEntityResponse	1✔
148	var page int = 0	1✔
149	for {	2✔
150	logger.Debug("listEntities", "page", page)	1✔
151	resp := client.	1✔
152	Get("/api/v1/catalog").	1✔
153	// Filters	1✔
154	SetQueryParam("includeArchived", archived).	1✔
155	SetQueryParam("types", types).	1✔
156	SetQueryParam("groups", groups).	1✔
157	// Options	1✔
158	SetQueryParam("yaml", "false").	1✔
159	SetQueryParam("includeMetadata", "true").	1✔
160	SetQueryParam("includeLinks", "true").	1✔
161	SetQueryParam("includeSlackChannels", "true").	1✔
162	SetQueryParam("includeOwners", "true").	1✔
163	SetQueryParam("includeHierarchyFields", "true").	1✔
164	// Pagination	1✔
165	SetQueryParam("pageSize", "1000").	1✔
166	SetQueryParam("page", strconv.Itoa(page)).	1✔
167	Do(ctx)	1✔
168		1✔
169	// Check for HTTP errors	1✔
170	if resp.IsErrorState() {	2✔
171	logger.Error("listEntities", "Status", resp.Status, "Body", resp.String())	1✔
172	return fmt.Errorf("error from cortex API %s: %s", resp.Status, resp.String())	1✔
173	}	1✔
174
175	// Unmarshal the response and check for unmarshal errors
176	err := resp.Into(&response)	1✔
177	if err != nil {	1✔
UNCOV 178	logger.Error("listEntities", "page", page, "Error", err)	×
UNCOV 179	return err	×
UNCOV 180	}	×
181
182	logger.Debug("listEntities", "totalPages", response.TotalPages, "total", response.Total)	1✔
183		1✔
184	for _, result := range response.Entities {	2✔
185	// send the item to steampipe	1✔
186	writer.StreamListItem(ctx, result)	1✔
187	// Context can be cancelled due to manual cancellation or the limit has been hit	1✔
188	if writer.RowsRemaining(ctx) == 0 {	1✔
UNCOV 189	logger.Debug("listEntities", "RowsRemaining", writer.RowsRemaining(ctx))	×
UNCOV 190	return nil	×
UNCOV 191	}	×
192	}
193	page++	1✔
194	if page >= response.TotalPages {	2✔
195	logger.Debug("listEntities", "page", page, "totalPages", response.TotalPages)	1✔
196	break	1✔
197	}
198	}
199	return nil	1✔
200	}
201
202	func buildGroupFilters(groupQuals []*quals.Qual) []string {	1✔
203	var groupFilters []string	1✔
204	for _, q := range groupQuals {	2✔
205	switch q.Operator {	1✔
206	case quals.QualOperatorJsonbExistsOne, quals.QualOperatorEqual:	1✔
207	if value := q.Value.GetStringValue(); value != "" {	2✔
208	groupFilters = append(groupFilters, value)	1✔
209	}	1✔
210	case quals.QualOperatorJsonbExistsAny:	1✔
211	if listValue := q.Value.GetListValue(); listValue != nil {	2✔
212	for _, v := range listValue.Values {	2✔
213	if value := v.GetStringValue(); value != "" {	2✔
214	groupFilters = append(groupFilters, value)	1✔
215	}	1✔
216	}
217	}
218	}
219	}
220	return groupFilters	1✔
221	}

Smirl / steampipe-plugin-cortex / 18127951005

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous