24593303471

Committed 18 Apr 2026 01:05AM UTC coverage: 74.959% (+2.3%) from 72.706%

Build # 24593303471

Build Type

push

github

Committed by

ravisuhag

Commit Message

fix: use GreaterOrEqual assertion for Presto system table count

The test used prestodb/presto:latest and asserted exactly 30 system
tables. Newer Presto versions add tables, breaking the exact match.
Use GreaterOrEqual to tolerate additions.

Coverage Stats

6403 of 8542 relevant lines covered (74.96%)

0.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.37

/plugins/extractors/github/github.go

package github

import (
        "context"
        _ "embed"
        "fmt"
        "path/filepath"
        "strings"

        gh "github.com/google/go-github/v68/github"
        "github.com/raystack/meteor/models"
        meteorv1beta1 "github.com/raystack/meteor/models/raystack/meteor/v1beta1"
        "github.com/raystack/meteor/plugins"
        "github.com/raystack/meteor/registry"
        log "github.com/raystack/salt/observability/logger"
        "golang.org/x/oauth2"
)

//go:embed README.md
var summary string

type Config struct {
        Org     string     `json:"org" yaml:"org" mapstructure:"org" validate:"required"`
        Token   string     `json:"token" yaml:"token" mapstructure:"token" validate:"required"`
        Extract []string   `json:"extract" yaml:"extract" mapstructure:"extract"`
        Docs    DocsConfig `json:"docs" yaml:"docs" mapstructure:"docs"`
}

type DocsConfig struct {
        Repos   []string `json:"repos" yaml:"repos" mapstructure:"repos"`
        Paths   []string `json:"paths" yaml:"paths" mapstructure:"paths"`
        Pattern string   `json:"pattern" yaml:"pattern" mapstructure:"pattern"`
}

var sampleConfig = `
org: raystack
token: github_token
# extract specifies which entity types to extract.
# Defaults to all: ["users", "repositories", "teams"]
extract:
  - users
  - repositories
  - teams
  - documents
# docs configures document extraction (only used when "documents" is in extract).
docs:
  # repos limits which repositories to scan. If empty, scans all org repos.
  repos: []
  # paths specifies directory paths to scan for documents. Defaults to ["docs"].
  paths:
    - docs
  # pattern is a glob pattern to match files. Defaults to "*.md".
  pattern: "*.md"`

var info = plugins.Info{
        Description:  "Extract metadata from a GitHub organisation including users, repositories, teams, and documents.",
        SampleConfig: sampleConfig,
        Summary:      summary,
        Tags:         []string{"platform", "extractor"},
}

type Extractor struct {
        plugins.BaseExtractor
        logger  log.Logger
        config  Config
        client  *gh.Client
        extract map[string]bool
}

func New(logger log.Logger) *Extractor {
        e := &Extractor{logger: logger}
        e.BaseExtractor = plugins.NewBaseExtractor(info, &e.config)
        return e
}

func (e *Extractor) Init(ctx context.Context, config plugins.Config) error {
        if err := e.BaseExtractor.Init(ctx, config); err != nil {
                return err
        }

        ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: e.config.Token})
        tc := oauth2.NewClient(ctx, ts)
        e.client = gh.NewClient(tc)

        e.extract = map[string]bool{
                "users":        true,
                "repositories": true,
                "teams":        true,
                "documents":    true,
        }
        if len(e.config.Extract) > 0 {
                e.extract = make(map[string]bool, len(e.config.Extract))
                for _, v := range e.config.Extract {
                        e.extract[v] = true
                }
        }

        return nil
}

// SetBaseURL overrides the GitHub API base URL (used for testing).
func (e *Extractor) SetBaseURL(url string) {
        e.client.BaseURL, _ = e.client.BaseURL.Parse(url + "/api/v3/")
}

func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) error {
        if e.extract["users"] {
                if err := e.extractUsers(ctx, emit); err != nil {
                        return fmt.Errorf("extract users: %w", err)
                }
        }
        if e.extract["repositories"] {
                if err := e.extractRepositories(ctx, emit); err != nil {
                        return fmt.Errorf("extract repositories: %w", err)
                }
        }
        if e.extract["teams"] {
                if err := e.extractTeams(ctx, emit); err != nil {
                        return fmt.Errorf("extract teams: %w", err)
                }
        }
        if e.extract["documents"] {
                if err := e.extractDocuments(ctx, emit); err != nil {
                        return fmt.Errorf("extract documents: %w", err)
                }
        }
        return nil
}

func (e *Extractor) extractUsers(ctx context.Context, emit plugins.Emit) error {
        opts := &gh.ListMembersOptions{
                ListOptions: gh.ListOptions{PerPage: 100},
        }
        for {
                members, resp, err := e.client.Organizations.ListMembers(ctx, e.config.Org, opts)
                if err != nil {
                        return fmt.Errorf("list members: %w", err)
                }

                for _, member := range members {
                        usr, _, err := e.client.Users.Get(ctx, member.GetLogin())
                        if err != nil {
                                e.logger.Warn("failed to fetch user, skipping", "login", member.GetLogin(), "error", err)
                                continue
                        }
                        emit(e.buildUserRecord(usr))
                }

                if resp.NextPage == 0 {
                        break
                }
                opts.Page = resp.NextPage
        }
        return nil
}

func (e *Extractor) buildUserRecord(usr *gh.User) models.Record {
        urn := models.NewURN("github", e.UrnScope, "user", usr.GetNodeID())
        props := map[string]any{
                "email":      usr.GetEmail(),
                "username":   usr.GetLogin(),
                "full_name":  usr.GetName(),
                "company":    usr.GetCompany(),
                "location":   usr.GetLocation(),
                "bio":        usr.GetBio(),
                "avatar_url": usr.GetAvatarURL(),
                "html_url":   usr.GetHTMLURL(),
                "status":     "active",
        }

        entity := models.NewEntity(urn, "user", usr.GetName(), "github", props)
        var edges []*meteorv1beta1.Edge
        edges = append(edges, &meteorv1beta1.Edge{
                SourceUrn: urn,
                TargetUrn: models.NewURN("github", e.UrnScope, "org", e.config.Org),
                Type:      "member_of",
                Source:    "github",
        })
        return models.NewRecord(entity, edges...)
}

func (e *Extractor) extractRepositories(ctx context.Context, emit plugins.Emit) error {
        opts := &gh.RepositoryListByOrgOptions{
                ListOptions: gh.ListOptions{PerPage: 100},
        }
        for {
                repos, resp, err := e.client.Repositories.ListByOrg(ctx, e.config.Org, opts)
                if err != nil {
                        return fmt.Errorf("list repositories: %w", err)
                }

                for _, repo := range repos {
                        emit(e.buildRepoRecord(repo))
                }

                if resp.NextPage == 0 {
                        break
                }
                opts.Page = resp.NextPage
        }
        return nil
}

func (e *Extractor) buildRepoRecord(repo *gh.Repository) models.Record {
        urn := models.NewURN("github", e.UrnScope, "repository", repo.GetNodeID())
        props := map[string]any{
                "full_name":     repo.GetFullName(),
                "description":   repo.GetDescription(),
                "html_url":      repo.GetHTMLURL(),
                "language":      repo.GetLanguage(),
                "visibility":    repo.GetVisibility(),
                "default_branch": repo.GetDefaultBranch(),
                "archived":      repo.GetArchived(),
                "fork":          repo.GetFork(),
                "stargazers":    repo.GetStargazersCount(),
                "forks":         repo.GetForksCount(),
                "open_issues":   repo.GetOpenIssuesCount(),
        }
        if len(repo.Topics) > 0 {
                props["topics"] = repo.Topics
        }

        entity := models.NewEntity(urn, "repository", repo.GetName(), "github", props)

        var edges []*meteorv1beta1.Edge
        if owner := repo.GetOwner(); owner != nil {
                edges = append(edges, models.OwnerEdge(
                        urn,
                        models.NewURN("github", e.UrnScope, "user", owner.GetNodeID()),
                        "github",
                ))
        }

        return models.NewRecord(entity, edges...)
}

func (e *Extractor) extractTeams(ctx context.Context, emit plugins.Emit) error {
        opts := &gh.ListOptions{PerPage: 100}
        for {
                teams, resp, err := e.client.Teams.ListTeams(ctx, e.config.Org, opts)
                if err != nil {
                        return fmt.Errorf("list teams: %w", err)
                }

                for _, team := range teams {
                        record, err := e.buildTeamRecord(ctx, team)
                        if err != nil {
                                e.logger.Warn("failed to build team record, skipping", "team", team.GetSlug(), "error", err)
                                continue
                        }
                        emit(record)
                }

                if resp.NextPage == 0 {
                        break
                }
                opts.Page = resp.NextPage
        }
        return nil
}

func (e *Extractor) buildTeamRecord(ctx context.Context, team *gh.Team) (models.Record, error) {
        urn := models.NewURN("github", e.UrnScope, "team", team.GetNodeID())
        props := map[string]any{
                "slug":        team.GetSlug(),
                "description": team.GetDescription(),
                "privacy":     team.GetPrivacy(),
                "permission":  team.GetPermission(),
                "html_url":    fmt.Sprintf("https://github.com/orgs/%s/teams/%s", e.config.Org, team.GetSlug()),
        }

        entity := models.NewEntity(urn, "team", team.GetName(), "github", props)

        var edges []*meteorv1beta1.Edge

        // Fetch team members and create member_of edges.
        memberOpts := &gh.TeamListTeamMembersOptions{
                ListOptions: gh.ListOptions{PerPage: 100},
        }
        for {
                members, resp, err := e.client.Teams.ListTeamMembersBySlug(ctx, e.config.Org, team.GetSlug(), memberOpts)
                if err != nil {
                        return models.Record{}, fmt.Errorf("list team members for %s: %w", team.GetSlug(), err)
                }

                for _, member := range members {
                        edges = append(edges, &meteorv1beta1.Edge{
                                SourceUrn: models.NewURN("github", e.UrnScope, "user", member.GetNodeID()),
                                TargetUrn: urn,
                                Type:      "member_of",
                                Source:    "github",
                        })
                }

                if resp.NextPage == 0 {
                        break
                }
                memberOpts.Page = resp.NextPage
        }

        return models.NewRecord(entity, edges...), nil
}

func (e *Extractor) extractDocuments(ctx context.Context, emit plugins.Emit) error {
        paths := e.config.Docs.Paths
        if len(paths) == 0 {
                paths = []string{"docs"}
        }
        pattern := e.config.Docs.Pattern
        if pattern == "" {
                pattern = "*.md"
        }

        repos, err := e.listDocRepos(ctx)
        if err != nil {
                return err
        }

        for _, repo := range repos {
                repoURN := models.NewURN("github", e.UrnScope, "repository", repo.GetNodeID())
                for _, dir := range paths {
                        if err := e.extractDocsFromPath(ctx, emit, repo, repoURN, dir, pattern); err != nil {
                                e.logger.Warn("failed to extract docs from path, skipping",
                                        "repo", repo.GetFullName(), "path", dir, "error", err)
                        }
                }
        }
        return nil
}

func (e *Extractor) listDocRepos(ctx context.Context) ([]*gh.Repository, error) {
        if len(e.config.Docs.Repos) > 0 {
                var repos []*gh.Repository
                for _, name := range e.config.Docs.Repos {
                        repo, _, err := e.client.Repositories.Get(ctx, e.config.Org, name)
                        if err != nil {
                                e.logger.Warn("failed to get repo for docs, skipping", "repo", name, "error", err)
                                continue
                        }
                        repos = append(repos, repo)
                }
                return repos, nil
        }

        // Fall back to all org repos.
        var all []*gh.Repository
        opts := &gh.RepositoryListByOrgOptions{
                ListOptions: gh.ListOptions{PerPage: 100},
        }
        for {
                repos, resp, err := e.client.Repositories.ListByOrg(ctx, e.config.Org, opts)
                if err != nil {
                        return nil, fmt.Errorf("list repositories for docs: %w", err)
                }
                all = append(all, repos...)
                if resp.NextPage == 0 {
                        break
                }
                opts.Page = resp.NextPage
        }
        return all, nil
}

func (e *Extractor) extractDocsFromPath(ctx context.Context, emit plugins.Emit, repo *gh.Repository, repoURN, dir, pattern string) error {
        _, dirContents, _, err := e.client.Repositories.GetContents(ctx, e.config.Org, repo.GetName(), dir, nil)
        if err != nil {
                return fmt.Errorf("get contents of %s: %w", dir, err)
        }

        for _, entry := range dirContents {
                switch entry.GetType() {
                case "file":
                        matched, _ := filepath.Match(pattern, entry.GetName())
                        if !matched {
                                continue
                        }
                        if err := e.emitDocument(ctx, emit, repo, repoURN, entry); err != nil {
                                e.logger.Warn("failed to emit document, skipping",
                                        "repo", repo.GetFullName(), "path", entry.GetPath(), "error", err)
                        }
                case "dir":
                        if err := e.extractDocsFromPath(ctx, emit, repo, repoURN, entry.GetPath(), pattern); err != nil {
                                e.logger.Warn("failed to recurse into directory, skipping",
                                        "repo", repo.GetFullName(), "path", entry.GetPath(), "error", err)
                        }
                }
        }
        return nil
}

func (e *Extractor) emitDocument(ctx context.Context, emit plugins.Emit, repo *gh.Repository, repoURN string, entry *gh.RepositoryContent) error {
        // Fetch full file content (the directory listing doesn't include content).
        file, _, _, err := e.client.Repositories.GetContents(ctx, e.config.Org, repo.GetName(), entry.GetPath(), nil)
        if err != nil {
                return fmt.Errorf("get file %s: %w", entry.GetPath(), err)
        }

        content, err := file.GetContent()
        if err != nil {
                return fmt.Errorf("decode content of %s: %w", entry.GetPath(), err)
        }

        name := strings.TrimSuffix(entry.GetName(), filepath.Ext(entry.GetName()))
        urn := models.NewURN("github", e.UrnScope, "document", file.GetSHA())

        props := map[string]any{
                "path":      file.GetPath(),
                "file_name": file.GetName(),
                "content":   content,
                "html_url":  file.GetHTMLURL(),
                "repo":      repo.GetFullName(),
                "size":      file.GetSize(),
                "sha":       file.GetSHA(),
        }

        entity := models.NewEntity(urn, "document", name, "github", props)

        edges := []*meteorv1beta1.Edge{
                {
                        SourceUrn: urn,
                        TargetUrn: repoURN,
                        Type:      "belongs_to",
                        Source:    "github",
                },
        }

        emit(models.NewRecord(entity, edges...))
        return nil
}

func init() {
        if err := registry.Extractors.Register("github", func() plugins.Extractor {
                return New(plugins.GetLog())
        }); err != nil {
                panic(err)
        }
}

1	package github
2
3	import (
4	"context"
5	_ "embed"
6	"fmt"
7	"path/filepath"
8	"strings"
9
10	gh "github.com/google/go-github/v68/github"
11	"github.com/raystack/meteor/models"
12	meteorv1beta1 "github.com/raystack/meteor/models/raystack/meteor/v1beta1"
13	"github.com/raystack/meteor/plugins"
14	"github.com/raystack/meteor/registry"
15	log "github.com/raystack/salt/observability/logger"
16	"golang.org/x/oauth2"
17	)
18
19	//go:embed README.md
20	var summary string
21
22	type Config struct {
23	Org string `json:"org" yaml:"org" mapstructure:"org" validate:"required"`
24	Token string `json:"token" yaml:"token" mapstructure:"token" validate:"required"`
25	Extract []string `json:"extract" yaml:"extract" mapstructure:"extract"`
26	Docs DocsConfig `json:"docs" yaml:"docs" mapstructure:"docs"`
27	}
28
29	type DocsConfig struct {
30	Repos []string `json:"repos" yaml:"repos" mapstructure:"repos"`
31	Paths []string `json:"paths" yaml:"paths" mapstructure:"paths"`
32	Pattern string `json:"pattern" yaml:"pattern" mapstructure:"pattern"`
33	}
34
35	var sampleConfig = `
36	org: raystack
37	token: github_token
38	# extract specifies which entity types to extract.
39	# Defaults to all: ["users", "repositories", "teams"]
40	extract:
41	- users
42	- repositories
43	- teams
44	- documents
45	# docs configures document extraction (only used when "documents" is in extract).
46	docs:
47	# repos limits which repositories to scan. If empty, scans all org repos.
48	repos: []
49	# paths specifies directory paths to scan for documents. Defaults to ["docs"].
50	paths:
51	- docs
52	# pattern is a glob pattern to match files. Defaults to "*.md".
53	pattern: "*.md"`
54
55	var info = plugins.Info{
56	Description: "Extract metadata from a GitHub organisation including users, repositories, teams, and documents.",
57	SampleConfig: sampleConfig,
58	Summary: summary,
59	Tags: []string{"platform", "extractor"},
60	}
61
62	type Extractor struct {
63	plugins.BaseExtractor
64	logger log.Logger
65	config Config
66	client *gh.Client
67	extract map[string]bool
68	}
69
70	func New(logger log.Logger) *Extractor {	1✔
71	e := &Extractor{logger: logger}	1✔
72	e.BaseExtractor = plugins.NewBaseExtractor(info, &e.config)	1✔
73	return e	1✔
74	}	1✔
75
76	func (e *Extractor) Init(ctx context.Context, config plugins.Config) error {	1✔
77	if err := e.BaseExtractor.Init(ctx, config); err != nil {	2✔
78	return err	1✔
79	}	1✔
80
81	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: e.config.Token})	1✔
82	tc := oauth2.NewClient(ctx, ts)	1✔
83	e.client = gh.NewClient(tc)	1✔
84		1✔
85	e.extract = map[string]bool{	1✔
86	"users": true,	1✔
87	"repositories": true,	1✔
88	"teams": true,	1✔
89	"documents": true,	1✔
90	}	1✔
91	if len(e.config.Extract) > 0 {	2✔
92	e.extract = make(map[string]bool, len(e.config.Extract))	1✔
93	for _, v := range e.config.Extract {	2✔
94	e.extract[v] = true	1✔
95	}	1✔
96	}
97
98	return nil	1✔
99	}
100
101	// SetBaseURL overrides the GitHub API base URL (used for testing).
102	func (e *Extractor) SetBaseURL(url string) {	1✔
103	e.client.BaseURL, _ = e.client.BaseURL.Parse(url + "/api/v3/")	1✔
104	}	1✔
105
106	func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) error {	1✔
107	if e.extract["users"] {	2✔
108	if err := e.extractUsers(ctx, emit); err != nil {	2✔
109	return fmt.Errorf("extract users: %w", err)	1✔
110	}	1✔
111	}
112	if e.extract["repositories"] {	2✔
113	if err := e.extractRepositories(ctx, emit); err != nil {	1✔
114	return fmt.Errorf("extract repositories: %w", err)	×
115	}	×
116	}
117	if e.extract["teams"] {	2✔
118	if err := e.extractTeams(ctx, emit); err != nil {	1✔
119	return fmt.Errorf("extract teams: %w", err)	×
120	}	×
121	}
122	if e.extract["documents"] {	2✔
123	if err := e.extractDocuments(ctx, emit); err != nil {	1✔
124	return fmt.Errorf("extract documents: %w", err)	×
125	}	×
126	}
127	return nil	1✔
128	}
129
130	func (e *Extractor) extractUsers(ctx context.Context, emit plugins.Emit) error {	1✔
131	opts := &gh.ListMembersOptions{	1✔
132	ListOptions: gh.ListOptions{PerPage: 100},	1✔
133	}	1✔
134	for {	2✔
135	members, resp, err := e.client.Organizations.ListMembers(ctx, e.config.Org, opts)	1✔
136	if err != nil {	2✔
137	return fmt.Errorf("list members: %w", err)	1✔
138	}	1✔
139
140	for _, member := range members {	2✔
141	usr, _, err := e.client.Users.Get(ctx, member.GetLogin())	1✔
142	if err != nil {	2✔
143	e.logger.Warn("failed to fetch user, skipping", "login", member.GetLogin(), "error", err)	1✔
144	continue	1✔
145	}
146	emit(e.buildUserRecord(usr))	1✔
147	}
148
149	if resp.NextPage == 0 {	2✔
150	break	1✔
151	}
152	opts.Page = resp.NextPage	1✔
153	}
154	return nil	1✔
155	}
156
157	func (e Extractor) buildUserRecord(usr gh.User) models.Record {	1✔
158	urn := models.NewURN("github", e.UrnScope, "user", usr.GetNodeID())	1✔
159	props := map[string]any{	1✔
160	"email": usr.GetEmail(),	1✔
161	"username": usr.GetLogin(),	1✔
162	"full_name": usr.GetName(),	1✔
163	"company": usr.GetCompany(),	1✔
164	"location": usr.GetLocation(),	1✔
165	"bio": usr.GetBio(),	1✔
166	"avatar_url": usr.GetAvatarURL(),	1✔
167	"html_url": usr.GetHTMLURL(),	1✔
168	"status": "active",	1✔
169	}	1✔
170		1✔
171	entity := models.NewEntity(urn, "user", usr.GetName(), "github", props)	1✔
172	var edges []*meteorv1beta1.Edge	1✔
173	edges = append(edges, &meteorv1beta1.Edge{	1✔
174	SourceUrn: urn,	1✔
175	TargetUrn: models.NewURN("github", e.UrnScope, "org", e.config.Org),	1✔
176	Type: "member_of",	1✔
177	Source: "github",	1✔
178	})	1✔
179	return models.NewRecord(entity, edges...)	1✔
180	}	1✔
181
182	func (e *Extractor) extractRepositories(ctx context.Context, emit plugins.Emit) error {	1✔
183	opts := &gh.RepositoryListByOrgOptions{	1✔
184	ListOptions: gh.ListOptions{PerPage: 100},	1✔
185	}	1✔
186	for {	2✔
187	repos, resp, err := e.client.Repositories.ListByOrg(ctx, e.config.Org, opts)	1✔
188	if err != nil {	1✔
189	return fmt.Errorf("list repositories: %w", err)	×
190	}	×
191
192	for _, repo := range repos {	2✔
193	emit(e.buildRepoRecord(repo))	1✔
194	}	1✔
195
196	if resp.NextPage == 0 {	2✔
197	break	1✔
198	}
199	opts.Page = resp.NextPage	×
200	}
201	return nil	1✔
202	}
203
204	func (e Extractor) buildRepoRecord(repo gh.Repository) models.Record {	1✔
205	urn := models.NewURN("github", e.UrnScope, "repository", repo.GetNodeID())	1✔
206	props := map[string]any{	1✔
207	"full_name": repo.GetFullName(),	1✔
208	"description": repo.GetDescription(),	1✔
209	"html_url": repo.GetHTMLURL(),	1✔
210	"language": repo.GetLanguage(),	1✔
211	"visibility": repo.GetVisibility(),	1✔
212	"default_branch": repo.GetDefaultBranch(),	1✔
213	"archived": repo.GetArchived(),	1✔
214	"fork": repo.GetFork(),	1✔
215	"stargazers": repo.GetStargazersCount(),	1✔
216	"forks": repo.GetForksCount(),	1✔
217	"open_issues": repo.GetOpenIssuesCount(),	1✔
218	}	1✔
219	if len(repo.Topics) > 0 {	2✔
220	props["topics"] = repo.Topics	1✔
221	}	1✔
222
223	entity := models.NewEntity(urn, "repository", repo.GetName(), "github", props)	1✔
224		1✔
225	var edges []*meteorv1beta1.Edge	1✔
226	if owner := repo.GetOwner(); owner != nil {	2✔
227	edges = append(edges, models.OwnerEdge(	1✔
228	urn,	1✔
229	models.NewURN("github", e.UrnScope, "user", owner.GetNodeID()),	1✔
230	"github",	1✔
231	))	1✔
232	}	1✔
233
234	return models.NewRecord(entity, edges...)	1✔
235	}
236
237	func (e *Extractor) extractTeams(ctx context.Context, emit plugins.Emit) error {	1✔
238	opts := &gh.ListOptions{PerPage: 100}	1✔
239	for {	2✔
240	teams, resp, err := e.client.Teams.ListTeams(ctx, e.config.Org, opts)	1✔
241	if err != nil {	1✔
242	return fmt.Errorf("list teams: %w", err)	×
243	}	×
244
245	for _, team := range teams {	2✔
246	record, err := e.buildTeamRecord(ctx, team)	1✔
247	if err != nil {	1✔
248	e.logger.Warn("failed to build team record, skipping", "team", team.GetSlug(), "error", err)	×
249	continue	×
250	}
251	emit(record)	1✔
252	}
253
254	if resp.NextPage == 0 {	2✔
255	break	1✔
256	}
257	opts.Page = resp.NextPage	×
258	}
259	return nil	1✔
260	}
261
262	func (e Extractor) buildTeamRecord(ctx context.Context, team gh.Team) (models.Record, error) {	1✔
263	urn := models.NewURN("github", e.UrnScope, "team", team.GetNodeID())	1✔
264	props := map[string]any{	1✔
265	"slug": team.GetSlug(),	1✔
266	"description": team.GetDescription(),	1✔
267	"privacy": team.GetPrivacy(),	1✔
268	"permission": team.GetPermission(),	1✔
269	"html_url": fmt.Sprintf("https://github.com/orgs/%s/teams/%s", e.config.Org, team.GetSlug()),	1✔
270	}	1✔
271		1✔
272	entity := models.NewEntity(urn, "team", team.GetName(), "github", props)	1✔
273		1✔
274	var edges []*meteorv1beta1.Edge	1✔
275		1✔
276	// Fetch team members and create member_of edges.	1✔
277	memberOpts := &gh.TeamListTeamMembersOptions{	1✔
278	ListOptions: gh.ListOptions{PerPage: 100},	1✔
279	}	1✔
280	for {	2✔
281	members, resp, err := e.client.Teams.ListTeamMembersBySlug(ctx, e.config.Org, team.GetSlug(), memberOpts)	1✔
282	if err != nil {	1✔
283	return models.Record{}, fmt.Errorf("list team members for %s: %w", team.GetSlug(), err)	×
284	}	×
285
286	for _, member := range members {	2✔
287	edges = append(edges, &meteorv1beta1.Edge{	1✔
288	SourceUrn: models.NewURN("github", e.UrnScope, "user", member.GetNodeID()),	1✔
289	TargetUrn: urn,	1✔
290	Type: "member_of",	1✔
291	Source: "github",	1✔
292	})	1✔
293	}	1✔
294
295	if resp.NextPage == 0 {	2✔
296	break	1✔
297	}
298	memberOpts.Page = resp.NextPage	×
299	}
300
301	return models.NewRecord(entity, edges...), nil	1✔
302	}
303
304	func (e *Extractor) extractDocuments(ctx context.Context, emit plugins.Emit) error {	1✔
305	paths := e.config.Docs.Paths	1✔
306	if len(paths) == 0 {	2✔
307	paths = []string{"docs"}	1✔
308	}	1✔
309	pattern := e.config.Docs.Pattern	1✔
310	if pattern == "" {	2✔
311	pattern = "*.md"	1✔
312	}	1✔
313
314	repos, err := e.listDocRepos(ctx)	1✔
315	if err != nil {	1✔
316	return err	×
317	}	×
318
319	for _, repo := range repos {	2✔
320	repoURN := models.NewURN("github", e.UrnScope, "repository", repo.GetNodeID())	1✔
321	for _, dir := range paths {	2✔
322	if err := e.extractDocsFromPath(ctx, emit, repo, repoURN, dir, pattern); err != nil {	2✔
323	e.logger.Warn("failed to extract docs from path, skipping",	1✔
324	"repo", repo.GetFullName(), "path", dir, "error", err)	1✔
325	}	1✔
326	}
327	}
328	return nil	1✔
329	}
330
331	func (e Extractor) listDocRepos(ctx context.Context) ([]gh.Repository, error) {	1✔
332	if len(e.config.Docs.Repos) > 0 {	2✔
333	var repos []*gh.Repository	1✔
334	for _, name := range e.config.Docs.Repos {	2✔
335	repo, _, err := e.client.Repositories.Get(ctx, e.config.Org, name)	1✔
336	if err != nil {	1✔
337	e.logger.Warn("failed to get repo for docs, skipping", "repo", name, "error", err)	×
338	continue	×
339	}
340	repos = append(repos, repo)	1✔
341	}
342	return repos, nil	1✔
343	}
344
345	// Fall back to all org repos.
346	var all []*gh.Repository	1✔
347	opts := &gh.RepositoryListByOrgOptions{	1✔
348	ListOptions: gh.ListOptions{PerPage: 100},	1✔
349	}	1✔
350	for {	2✔
351	repos, resp, err := e.client.Repositories.ListByOrg(ctx, e.config.Org, opts)	1✔
352	if err != nil {	1✔
353	return nil, fmt.Errorf("list repositories for docs: %w", err)	×
354	}	×
355	all = append(all, repos...)	1✔
356	if resp.NextPage == 0 {	2✔
357	break	1✔
358	}
359	opts.Page = resp.NextPage	×
360	}
361	return all, nil	1✔
362	}
363
364	func (e Extractor) extractDocsFromPath(ctx context.Context, emit plugins.Emit, repo gh.Repository, repoURN, dir, pattern string) error {	1✔
365	_, dirContents, _, err := e.client.Repositories.GetContents(ctx, e.config.Org, repo.GetName(), dir, nil)	1✔
366	if err != nil {	2✔
367	return fmt.Errorf("get contents of %s: %w", dir, err)	1✔
368	}	1✔
369
370	for _, entry := range dirContents {	2✔
371	switch entry.GetType() {	1✔
372	case "file":	1✔
373	matched, _ := filepath.Match(pattern, entry.GetName())	1✔
374	if !matched {	2✔
375	continue	1✔
376	}
377	if err := e.emitDocument(ctx, emit, repo, repoURN, entry); err != nil {	1✔
378	e.logger.Warn("failed to emit document, skipping",	×
379	"repo", repo.GetFullName(), "path", entry.GetPath(), "error", err)	×
380	}	×
381	case "dir":	1✔
382	if err := e.extractDocsFromPath(ctx, emit, repo, repoURN, entry.GetPath(), pattern); err != nil {	1✔
383	e.logger.Warn("failed to recurse into directory, skipping",	×
384	"repo", repo.GetFullName(), "path", entry.GetPath(), "error", err)	×
385	}	×
386	}
387	}
388	return nil	1✔
389	}
390
391	func (e Extractor) emitDocument(ctx context.Context, emit plugins.Emit, repo gh.Repository, repoURN string, entry *gh.RepositoryContent) error {	1✔
392	// Fetch full file content (the directory listing doesn't include content).	1✔
393	file, _, _, err := e.client.Repositories.GetContents(ctx, e.config.Org, repo.GetName(), entry.GetPath(), nil)	1✔
394	if err != nil {	1✔
395	return fmt.Errorf("get file %s: %w", entry.GetPath(), err)	×
396	}	×
397
398	content, err := file.GetContent()	1✔
399	if err != nil {	1✔
400	return fmt.Errorf("decode content of %s: %w", entry.GetPath(), err)	×
401	}	×
402
403	name := strings.TrimSuffix(entry.GetName(), filepath.Ext(entry.GetName()))	1✔
404	urn := models.NewURN("github", e.UrnScope, "document", file.GetSHA())	1✔
405		1✔
406	props := map[string]any{	1✔
407	"path": file.GetPath(),	1✔
408	"file_name": file.GetName(),	1✔
409	"content": content,	1✔
410	"html_url": file.GetHTMLURL(),	1✔
411	"repo": repo.GetFullName(),	1✔
412	"size": file.GetSize(),	1✔
413	"sha": file.GetSHA(),	1✔
414	}	1✔
415		1✔
416	entity := models.NewEntity(urn, "document", name, "github", props)	1✔
417		1✔
418	edges := []*meteorv1beta1.Edge{	1✔
419	{	1✔
420	SourceUrn: urn,	1✔
421	TargetUrn: repoURN,	1✔
422	Type: "belongs_to",	1✔
423	Source: "github",	1✔
424	},	1✔
425	}	1✔
426		1✔
427	emit(models.NewRecord(entity, edges...))	1✔
428	return nil	1✔
429	}
430
431	func init() {	1✔
432	if err := registry.Extractors.Register("github", func() plugins.Extractor {	1✔
433	return New(plugins.GetLog())	×
434	}); err != nil {	×
435	panic(err)	×
436	}
437	}

raystack / meteor / 24593303471

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous