13975345119

Committed 20 Mar 2025 05:13PM UTC coverage: 92.629% (-1.2%) from 93.786%

Build # 13975345119

Build Type

Pull #68

github

Committed by

robertodauria

Commit Message

Restore Cloud API keys creation, set datastore key to same value

Pull Request Pull Request #68: Restore Cloud API keys creation, set datastore key to same value

Run Details

32 of 43 new or added lines in 3 files covered. (74.42%)

19 existing lines in 2 files now uncovered.

1307 of 1411 relevant lines covered (92.63%)

1.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.1

/internal/adminx/org.go

package adminx

import (
        "context"
        "fmt"
        "log"

        "github.com/m-lab/autojoin/internal/dnsname"
        "golang.org/x/exp/slices"

        "google.golang.org/api/cloudresourcemanager/v1"
        "google.golang.org/api/dns/v1"
        "google.golang.org/api/iam/v1"
)

var (
        // Restrict uploads to the organization prefix. Needed to share bucket write access.
        expUploadFmt = (`resource.name.startsWith("projects/_/buckets/archive-%s/objects/autoload/v2/%s") ||` +
                ` resource.name.startsWith("projects/_/buckets/staging-%s/objects/autoload/v2/%s")`)
        // Restrict reads to the archive bucket. Needed so nodes can read jostler schemas.
        expReadFmt = (`resource.name.startsWith("projects/_/buckets/archive-%s") ||` +
                ` resource.name.startsWith("projects/_/buckets/downloader-%s") ||` +
                ` resource.name.startsWith("projects/_/buckets/staging-%s")`)

        // Allow uploads to include tables. Needed for the authoritative schema update path.
        expUploadTablesFmt = (`resource.name.startsWith("projects/_/buckets/archive-%s/objects/autoload/v2/%s") ||` +
                ` resource.name.startsWith("projects/_/buckets/staging-%s/objects/autoload/v2/%s") ||` +
                ` resource.name.startsWith("projects/_/buckets/archive-%s/objects/autoload/v2/tables") ||` +
                ` resource.name.startsWith("projects/_/buckets/staging-%s/objects/autoload/v2/tables")`)
)

// DNS is a simplified interface to the Google Cloud DNS API.
type DNS interface {
        RegisterZone(ctx context.Context, zone *dns.ManagedZone) (*dns.ManagedZone, error)
        RegisterZoneSplit(ctx context.Context, zone *dns.ManagedZone) (*dns.ResourceRecordSet, error)
}

// CRM is a simplified interface to the Google Cloud Resource Manager API.
type CRM interface {
        GetIamPolicy(ctx context.Context, req *cloudresourcemanager.GetIamPolicyRequest) (*cloudresourcemanager.Policy, error)
        SetIamPolicy(ctx context.Context, req *cloudresourcemanager.SetIamPolicyRequest) error
}

// OrganizationManager defines the interface for managing organizations and their API keys
type OrganizationManager interface {
        CreateOrganization(ctx context.Context, name, email string) error
        CreateAPIKeyWithValue(ctx context.Context, org, value string) (string, error)
        GetAPIKeys(ctx context.Context, org string) ([]string, error)
}

// Keys is the interface used to manage organization API keys.
type Keys interface {
        CreateKey(ctx context.Context, org string) (string, error)
}

// Org contains fields needed to setup a new organization for Autojoined nodes.
type Org struct {
        Project      string
        crm          CRM
        sam          *ServiceAccountsManager
        sm           *SecretManager
        orgm         OrganizationManager
        dns          DNS
        keys         Keys
        updateTables bool
}

// NewOrg creates a new Org instance for setting up a new organization.
func NewOrg(project string, crm CRM, sam *ServiceAccountsManager, sm *SecretManager, dns DNS, k Keys,
        orgm OrganizationManager, updateTables bool) *Org {
        return &Org{
                Project:      project,
                crm:          crm,
                sam:          sam,
                sm:           sm,
                orgm:         orgm,
                dns:          dns,
                keys:         k,
                updateTables: updateTables,
        }
}

// Setup should be run once on org creation to create all Google Cloud resources needed by the Autojoin API.
func (o *Org) Setup(ctx context.Context, org string, email string) error {
        // Create organization in Datastore
        err := o.orgm.CreateOrganization(ctx, org, email)
        if err != nil {
                return err
        }
        // Create service account with no keys.
        sa, err := o.sam.CreateServiceAccount(ctx, org)
        if err != nil {
                return err
        }
        err = o.ApplyPolicy(ctx, org, sa, o.updateTables)
        if err != nil {
                return err
        }
        // Create secret with no versions.
        err = o.sm.CreateSecret(ctx, org)
        if err != nil {
                return err
        }
        // Create DNS zone and zone split.
        err = o.RegisterDNS(ctx, org)
        if err != nil {
                return err
        }
        return nil
}

// RegisterDNS creates the organization zone and the zone split within the project zone.
func (o *Org) RegisterDNS(ctx context.Context, org string) error {
        zone, err := o.dns.RegisterZone(ctx, &dns.ManagedZone{
                Description: "Autojoin registered nodes from org: " + org,
                Name:        dnsname.OrgZone(org, o.Project),
                DnsName:     dnsname.OrgDNS(org, o.Project),
                DnssecConfig: &dns.ManagedZoneDnsSecConfig{
                        State: "on",
                },
        })
        if err != nil {
                log.Println("failed to register zone:", dnsname.OrgZone(org, o.Project), err)
                return err
        }
        _, err = o.dns.RegisterZoneSplit(ctx, zone)
        if err != nil {
                log.Println("failed to register zone split:", dnsname.OrgZone(org, o.Project), err)
                return err
        }
        return nil
}

// ApplyPolicy adds write restrictions for shared GCS buckets.
// NOTE: By operating on project IAM policies, this method modifies project wide state.
func (o *Org) ApplyPolicy(ctx context.Context, org string, account *iam.ServiceAccount, updateTables bool) error {
        // Get current policy.
        req := &cloudresourcemanager.GetIamPolicyRequest{
                Options: &cloudresourcemanager.GetPolicyOptions{
                        RequestedPolicyVersion: 3,
                },
        }
        curr, err := o.crm.GetIamPolicy(ctx, req)
        if err != nil {
                log.Println("get policy", err)
                return err
        }
        expression := ""
        role := ""
        if updateTables {
                // Allow this role to upload data and update schema tables.
                expression = fmt.Sprintf(expUploadTablesFmt, o.Project, org, o.Project, org, o.Project, o.Project)
                role = "roles/storage.objectUser"
        } else {
                // Only allow this role to upload data.
                expression = fmt.Sprintf(expUploadFmt, o.Project, org, o.Project, org)
                role = "roles/storage.objectCreator"
        }
        // Setup new bindings.
        bindings := []*cloudresourcemanager.Binding{
                {
                        Condition: &cloudresourcemanager.Expr{
                                Title:      "Upload restriction for " + org,
                                Expression: expression,
                        },
                        Members: []string{"serviceAccount:" + account.Email},
                        Role:    role,
                },
                {
                        Condition: &cloudresourcemanager.Expr{
                                Title:      "Read restriction for " + org,
                                Expression: fmt.Sprintf(expReadFmt, o.Project, o.Project, o.Project),
                        },
                        Members: []string{"serviceAccount:" + account.Email},
                        Role:    "roles/storage.objectViewer",
                },
        }

        // Append the new bindings if missing from the current set.
        newBindings, wasMissing := appendBindingIfMissing(curr.Bindings, bindings...)

        // Apply bindings if any were missing.
        preq := &cloudresourcemanager.SetIamPolicyRequest{
                Policy: &cloudresourcemanager.Policy{
                        AuditConfigs: curr.AuditConfigs,
                        Bindings:     newBindings,
                        Etag:         curr.Etag,
                        Version:      curr.Version,
                },
        }

        if wasMissing {
                err = o.crm.SetIamPolicy(ctx, preq)
                if err != nil {
                        log.Println("set policy", err)
                        return err
                }
        }
        return nil
}

func appendBindingIfMissing(slice []*cloudresourcemanager.Binding, elems ...*cloudresourcemanager.Binding) ([]*cloudresourcemanager.Binding, bool) {
        result := []*cloudresourcemanager.Binding{}
        foundMissing := false
        for _, b := range elems {
                found := false
                // Does slice contain B?
                for _, a := range slice {
                        if BindingIsEqual(a, b) {
                                // We found a matching binding.
                                found = true
                                break
                        }
                }
                if !found {
                        // slice does not contain B, so add it to results.
                        result = append(result, b)
                        foundMissing = true
                }
        }
        // Return all bindings
        return append(result, slice...), foundMissing
}

// BindingIsEqual checks wether the two provided bindings contain equal conditions, members, and roles.
func BindingIsEqual(a *cloudresourcemanager.Binding, b *cloudresourcemanager.Binding) bool {
        if (a.Condition != nil) != (b.Condition != nil) {
                // Either both should have conditions or neither.
                return false
        }
        if a.Condition != nil {
                // We established above that both are non-nil.
                if a.Condition.Expression != b.Condition.Expression {
                        // Expressions should match.
                        return false
                }
        }
        // Check membership in both directions: are all members of a in b, and b in a?
        for i := range a.Members {
                if !slices.Contains(b.Members, a.Members[i]) {
                        // Each member in A should be found in B.
                        return false
                }
        }
        for i := range b.Members {
                if !slices.Contains(a.Members, b.Members[i]) {
                        // Each member in B should be found in A.
                        return false
                }
        }
        // Roles should match.
        return a.Role == b.Role
}

func (o *Org) CreateAPIKeyWithValue(ctx context.Context, org, val string) (string, error) {
        return o.orgm.CreateAPIKeyWithValue(ctx, org, val)
}

1	package adminx
2
3	import (
4	"context"
5	"fmt"
6	"log"
7
8	"github.com/m-lab/autojoin/internal/dnsname"
9	"golang.org/x/exp/slices"
10
11	"google.golang.org/api/cloudresourcemanager/v1"
12	"google.golang.org/api/dns/v1"
13	"google.golang.org/api/iam/v1"
14	)
15
16	var (
17	// Restrict uploads to the organization prefix. Needed to share bucket write access.
18	expUploadFmt = (`resource.name.startsWith("projects/_/buckets/archive-%s/objects/autoload/v2/%s") \|\|` +
19	` resource.name.startsWith("projects/_/buckets/staging-%s/objects/autoload/v2/%s")`)
20	// Restrict reads to the archive bucket. Needed so nodes can read jostler schemas.
21	expReadFmt = (`resource.name.startsWith("projects/_/buckets/archive-%s") \|\|` +
22	` resource.name.startsWith("projects/_/buckets/downloader-%s") \|\|` +
23	` resource.name.startsWith("projects/_/buckets/staging-%s")`)
24
25	// Allow uploads to include tables. Needed for the authoritative schema update path.
26	expUploadTablesFmt = (`resource.name.startsWith("projects/_/buckets/archive-%s/objects/autoload/v2/%s") \|\|` +
27	` resource.name.startsWith("projects/_/buckets/staging-%s/objects/autoload/v2/%s") \|\|` +
28	` resource.name.startsWith("projects/_/buckets/archive-%s/objects/autoload/v2/tables") \|\|` +
29	` resource.name.startsWith("projects/_/buckets/staging-%s/objects/autoload/v2/tables")`)
30	)
31
32	// DNS is a simplified interface to the Google Cloud DNS API.
33	type DNS interface {
34	RegisterZone(ctx context.Context, zone dns.ManagedZone) (dns.ManagedZone, error)
35	RegisterZoneSplit(ctx context.Context, zone dns.ManagedZone) (dns.ResourceRecordSet, error)
36	}
37
38	// CRM is a simplified interface to the Google Cloud Resource Manager API.
39	type CRM interface {
40	GetIamPolicy(ctx context.Context, req cloudresourcemanager.GetIamPolicyRequest) (cloudresourcemanager.Policy, error)
41	SetIamPolicy(ctx context.Context, req *cloudresourcemanager.SetIamPolicyRequest) error
42	}
43
44	// OrganizationManager defines the interface for managing organizations and their API keys
45	type OrganizationManager interface {
46	CreateOrganization(ctx context.Context, name, email string) error
47	CreateAPIKeyWithValue(ctx context.Context, org, value string) (string, error)
48	GetAPIKeys(ctx context.Context, org string) ([]string, error)
49	}
50
51	// Keys is the interface used to manage organization API keys.
52	type Keys interface {
53	CreateKey(ctx context.Context, org string) (string, error)
54	}
55
56	// Org contains fields needed to setup a new organization for Autojoined nodes.
57	type Org struct {
58	Project string
59	crm CRM
60	sam *ServiceAccountsManager
61	sm *SecretManager
62	orgm OrganizationManager
63	dns DNS
64	keys Keys
65	updateTables bool
66	}
67
68	// NewOrg creates a new Org instance for setting up a new organization.
69	func NewOrg(project string, crm CRM, sam ServiceAccountsManager, sm SecretManager, dns DNS, k Keys,
70	orgm OrganizationManager, updateTables bool) *Org {	1✔
71	return &Org{	1✔
72	Project: project,	1✔
73	crm: crm,	1✔
74	sam: sam,	1✔
75	sm: sm,	1✔
76	orgm: orgm,	1✔
77	dns: dns,	1✔
78	keys: k,	1✔
79	updateTables: updateTables,	1✔
80	}	1✔
81	}	1✔
82
83	// Setup should be run once on org creation to create all Google Cloud resources needed by the Autojoin API.
84	func (o *Org) Setup(ctx context.Context, org string, email string) error {	1✔
85	// Create organization in Datastore	1✔
86	err := o.orgm.CreateOrganization(ctx, org, email)	1✔
87	if err != nil {	2✔
88	return err	1✔
89	}	1✔
90	// Create service account with no keys.
91	sa, err := o.sam.CreateServiceAccount(ctx, org)	1✔
92	if err != nil {	2✔
93	return err	1✔
94	}	1✔
95	err = o.ApplyPolicy(ctx, org, sa, o.updateTables)	1✔
96	if err != nil {	2✔
97	return err	1✔
98	}	1✔
99	// Create secret with no versions.
100	err = o.sm.CreateSecret(ctx, org)	1✔
101	if err != nil {	2✔
102	return err	1✔
103	}	1✔
104	// Create DNS zone and zone split.
105	err = o.RegisterDNS(ctx, org)	1✔
106	if err != nil {	2✔
107	return err	1✔
108	}	1✔
109	return nil	1✔
110	}
111
112	// RegisterDNS creates the organization zone and the zone split within the project zone.
113	func (o *Org) RegisterDNS(ctx context.Context, org string) error {	1✔
114	zone, err := o.dns.RegisterZone(ctx, &dns.ManagedZone{	1✔
115	Description: "Autojoin registered nodes from org: " + org,	1✔
116	Name: dnsname.OrgZone(org, o.Project),	1✔
117	DnsName: dnsname.OrgDNS(org, o.Project),	1✔
118	DnssecConfig: &dns.ManagedZoneDnsSecConfig{	1✔
119	State: "on",	1✔
120	},	1✔
121	})	1✔
122	if err != nil {	2✔
123	log.Println("failed to register zone:", dnsname.OrgZone(org, o.Project), err)	1✔
124	return err	1✔
125	}	1✔
126	_, err = o.dns.RegisterZoneSplit(ctx, zone)	1✔
127	if err != nil {	2✔
128	log.Println("failed to register zone split:", dnsname.OrgZone(org, o.Project), err)	1✔
129	return err	1✔
130	}	1✔
131	return nil	1✔
132	}
133
134	// ApplyPolicy adds write restrictions for shared GCS buckets.
135	// NOTE: By operating on project IAM policies, this method modifies project wide state.
136	func (o Org) ApplyPolicy(ctx context.Context, org string, account iam.ServiceAccount, updateTables bool) error {	1✔
137	// Get current policy.	1✔
138	req := &cloudresourcemanager.GetIamPolicyRequest{	1✔
139	Options: &cloudresourcemanager.GetPolicyOptions{	1✔
140	RequestedPolicyVersion: 3,	1✔
141	},	1✔
142	}	1✔
143	curr, err := o.crm.GetIamPolicy(ctx, req)	1✔
144	if err != nil {	2✔
145	log.Println("get policy", err)	1✔
146	return err	1✔
147	}	1✔
148	expression := ""	1✔
149	role := ""	1✔
150	if updateTables {	2✔
151	// Allow this role to upload data and update schema tables.	1✔
152	expression = fmt.Sprintf(expUploadTablesFmt, o.Project, org, o.Project, org, o.Project, o.Project)	1✔
153	role = "roles/storage.objectUser"	1✔
154	} else {	2✔
155	// Only allow this role to upload data.	1✔
156	expression = fmt.Sprintf(expUploadFmt, o.Project, org, o.Project, org)	1✔
157	role = "roles/storage.objectCreator"	1✔
158	}	1✔
159	// Setup new bindings.
160	bindings := []*cloudresourcemanager.Binding{	1✔
161	{	1✔
162	Condition: &cloudresourcemanager.Expr{	1✔
163	Title: "Upload restriction for " + org,	1✔
164	Expression: expression,	1✔
165	},	1✔
166	Members: []string{"serviceAccount:" + account.Email},	1✔
167	Role: role,	1✔
168	},	1✔
169	{	1✔
170	Condition: &cloudresourcemanager.Expr{	1✔
171	Title: "Read restriction for " + org,	1✔
172	Expression: fmt.Sprintf(expReadFmt, o.Project, o.Project, o.Project),	1✔
173	},	1✔
174	Members: []string{"serviceAccount:" + account.Email},	1✔
175	Role: "roles/storage.objectViewer",	1✔
176	},	1✔
177	}	1✔
178		1✔
179	// Append the new bindings if missing from the current set.	1✔
180	newBindings, wasMissing := appendBindingIfMissing(curr.Bindings, bindings...)	1✔
181		1✔
182	// Apply bindings if any were missing.	1✔
183	preq := &cloudresourcemanager.SetIamPolicyRequest{	1✔
184	Policy: &cloudresourcemanager.Policy{	1✔
185	AuditConfigs: curr.AuditConfigs,	1✔
186	Bindings: newBindings,	1✔
187	Etag: curr.Etag,	1✔
188	Version: curr.Version,	1✔
189	},	1✔
190	}	1✔
191		1✔
192	if wasMissing {	2✔
193	err = o.crm.SetIamPolicy(ctx, preq)	1✔
194	if err != nil {	2✔
195	log.Println("set policy", err)	1✔
196	return err	1✔
197	}	1✔
198	}
199	return nil	1✔
200	}
201
202	func appendBindingIfMissing(slice []cloudresourcemanager.Binding, elems ...cloudresourcemanager.Binding) ([]*cloudresourcemanager.Binding, bool) {	1✔
203	result := []*cloudresourcemanager.Binding{}	1✔
204	foundMissing := false	1✔
205	for _, b := range elems {	2✔
206	found := false	1✔
207	// Does slice contain B?	1✔
208	for _, a := range slice {	2✔
209	if BindingIsEqual(a, b) {	2✔
210	// We found a matching binding.	1✔
211	found = true	1✔
212	break	1✔
213	}
214	}
215	if !found {	2✔
216	// slice does not contain B, so add it to results.	1✔
217	result = append(result, b)	1✔
218	foundMissing = true	1✔
219	}	1✔
220	}
221	// Return all bindings
222	return append(result, slice...), foundMissing	1✔
223	}
224
225	// BindingIsEqual checks wether the two provided bindings contain equal conditions, members, and roles.
226	func BindingIsEqual(a cloudresourcemanager.Binding, b cloudresourcemanager.Binding) bool {	1✔
227	if (a.Condition != nil) != (b.Condition != nil) {	2✔
228	// Either both should have conditions or neither.	1✔
229	return false	1✔
230	}	1✔
231	if a.Condition != nil {	2✔
232	// We established above that both are non-nil.	1✔
233	if a.Condition.Expression != b.Condition.Expression {	2✔
234	// Expressions should match.	1✔
235	return false	1✔
236	}	1✔
237	}
238	// Check membership in both directions: are all members of a in b, and b in a?
239	for i := range a.Members {	2✔
240	if !slices.Contains(b.Members, a.Members[i]) {	2✔
241	// Each member in A should be found in B.	1✔
242	return false	1✔
243	}	1✔
244	}
245	for i := range b.Members {	2✔
246	if !slices.Contains(a.Members, b.Members[i]) {	2✔
247	// Each member in B should be found in A.	1✔
248	return false	1✔
249	}	1✔
250	}
251	// Roles should match.
252	return a.Role == b.Role	1✔
253	}
254
NEW 255	func (o *Org) CreateAPIKeyWithValue(ctx context.Context, org, val string) (string, error) {	×
NEW 256	return o.orgm.CreateAPIKeyWithValue(ctx, org, val)	×
UNCOV 257	}	×

m-lab / autojoin / 13975345119

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous