8299721104

Committed 15 Mar 2024 04:56PM UTC coverage: 19.95% (+0.5%) from 19.474%

Build # 8299721104

Build Type

Pull #96

github

Committed by

paulineribeyre

Commit Message

master - resolve conflicts

Pull Request Pull Request #96: MIDRC-543 Nextflow image builder AMI

Run Details

45 of 73 new or added lines in 3 files covered. (61.64%)

1 existing line in 1 file now uncovered.

1349 of 6762 relevant lines covered (19.95%)

1.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

28.71

/hatchery/helpers.go

package hatchery

import (
        "bytes"
        "context"
        "encoding/json"
        "errors"
        "fmt"
        "net/http"
        "os"
        "regexp"
        "strconv"
        "strings"
        "time"

        "github.com/aws/aws-sdk-go/aws"
        "github.com/aws/aws-sdk-go/aws/credentials/stscreds"
        "github.com/aws/aws-sdk-go/aws/session"
        "github.com/aws/aws-sdk-go/service/imagebuilder"
        "github.com/aws/aws-sdk-go/service/sts"
)

type APIKeyStruct struct {
        APIKey string `json:"api_key"`
        KeyID  string `json:"key_id"`
}

type WorkspaceKernelStatusStruct struct {
        LastActivityTime string `json:"last_activity"`
}

func StrToInt(str string) (string, error) {
        nonFractionalPart := strings.Split(str, ".")
        return nonFractionalPart[0], nil
}

func mem(str string) (string, error) {
        res := regexp.MustCompile(`(\d*)([M|G])ib?`)
        matches := res.FindStringSubmatch(str)
        num, err := strconv.Atoi(matches[1])
        if err != nil {
                return "", err
        }
        if matches[2] == "G" {
                num = num * 1024
        }
        return strconv.Itoa(num), nil
}

func cpu(str string) (string, error) {
        num, err := strconv.Atoi(str[:strings.IndexByte(str, '.')])
        if err != nil {
                return "", err
        }
        num = num * 1024
        return strconv.Itoa(num), nil
}

// Escapism escapes characters not allowed into hex with -
func escapism(input string) string {
        safeBytes := "abcdefghijklmnopqrstuvwxyz0123456789"
        var escaped string
        for _, v := range input {
                if !characterInString(v, safeBytes) {
                        hexCode := fmt.Sprintf("%2x", v)
                        escaped += "-" + hexCode
                } else {
                        escaped += string(v)
                }
        }
        return escaped
}

func characterInString(a rune, list string) bool {
        for _, b := range list {
                if b == a {
                        return true
                }
        }
        return false
}

func truncateString(str string, num int) string {
        bnoden := str
        if len(str) > num {
                bnoden = str[0:num]
        }
        if bnoden[len(bnoden)-1] == '-' {
                bnoden = bnoden[0 : len(bnoden)-2]
        }
        return bnoden
}

// API key related helper functions
// Make http request with header and body
func MakeARequestWithContext(ctx context.Context, method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer) (*http.Response, error) {
        if headers == nil {
                headers = make(map[string]string)
        }
        if accessToken != "" {
                headers["Authorization"] = "Bearer " + accessToken
        }
        if contentType != "" {
                headers["Content-Type"] = contentType
        }
        client := &http.Client{Timeout: 10 * time.Second}
        var req *http.Request
        var err error
        if body == nil {
                req, err = http.NewRequestWithContext(ctx, method, apiEndpoint, nil)
        } else {
                req, err = http.NewRequestWithContext(ctx, method, apiEndpoint, body)
        }

        if err != nil {
                return nil, errors.New("Error occurred during generating HTTP request: " + err.Error())
        }
        for k, v := range headers {
                req.Header.Add(k, v)
        }
        resp, err := client.Do(req)
        if err != nil {
                return nil, errors.New("Error occurred during making HTTP request: " + err.Error())
        }
        return resp, nil
}

func getFenceURL() string {
        fenceURL := "http://fence-service/"
        _, ok := os.LookupEnv("GEN3_ENDPOINT")
        if ok {
                fenceURL = "https://" + os.Getenv("GEN3_ENDPOINT") + "/user/"
        }
        return fenceURL
}

func getAmbassadorURL() string {
        ambassadorURL := "http://ambassador-service/"
        _, ok := os.LookupEnv("GEN3_ENDPOINT")
        if ok {
                ambassadorURL = "https://" + os.Getenv("GEN3_ENDPOINT") + "/lw-workspace/proxy/"
        }
        return ambassadorURL
}

func getAPIKeyWithContext(ctx context.Context, accessToken string) (apiKey *APIKeyStruct, err error) {
        if accessToken == "" {
                return nil, errors.New("No valid access token")
        }

        fenceAPIKeyURL := getFenceURL() + "credentials/api/"
        body := bytes.NewBufferString("{\"scope\": [\"data\", \"user\"]}")

        resp, err := MakeARequestWithContext(ctx, "POST", fenceAPIKeyURL, accessToken, "application/json", nil, body)
        if err != nil {
                return nil, err
        }

        if resp != nil && resp.StatusCode != 200 {
                return nil, errors.New("Error occurred when creating API key with error code " + strconv.Itoa(resp.StatusCode))
        }
        defer resp.Body.Close()

        fenceApiKeyResponse := new(APIKeyStruct)
        err = json.NewDecoder(resp.Body).Decode(fenceApiKeyResponse)
        if err != nil {
                return nil, errors.New("Unable to decode API key response: " + err.Error())
        }
        return fenceApiKeyResponse, nil
}

func deleteAPIKeyWithContext(ctx context.Context, accessToken string, apiKeyID string) error {
        if accessToken == "" {
                return errors.New("No valid access token")
        }

        fenceDeleteAPIKeyURL := getFenceURL() + "credentials/api/" + apiKeyID
        resp, err := MakeARequestWithContext(ctx, "DELETE", fenceDeleteAPIKeyURL, accessToken, "", nil, nil)
        if err != nil {
                return err
        }
        if resp != nil && resp.StatusCode != 204 {
                return errors.New("Error occurred when deleting API key with error code " + strconv.Itoa(resp.StatusCode))
        }
        return nil
}

func getKernelIdleTimeWithContext(ctx context.Context, accessToken string) (lastActivityTime int64, err error) {
        if accessToken == "" {
                return -1, errors.New("No valid access token")
        }

        workspaceKernelStatusURL := getAmbassadorURL() + "api/status"
        resp, err := MakeARequestWithContext(ctx, "GET", workspaceKernelStatusURL, accessToken, "", nil, nil)
        if err != nil {
                return -1, err
        }
        if resp != nil && resp.StatusCode != 200 {
                return -1, errors.New("Error occurred when getting workspace kernel status with error code " + strconv.Itoa(resp.StatusCode))
        }
        defer resp.Body.Close()

        workspaceKernelStatusResponse := new(WorkspaceKernelStatusStruct)
        err = json.NewDecoder(resp.Body).Decode(workspaceKernelStatusResponse)
        if err != nil {
                return -1, errors.New("Unable to decode workspace kernel status response: " + err.Error())
        }
        lastAct, err := time.Parse(time.RFC3339, workspaceKernelStatusResponse.LastActivityTime)
        if err != nil {
                return -1, errors.New("Unable to parse last activity time: " + err.Error())
        }
        return lastAct.Unix() * 1000, nil
}

func getAwsAccountId(sess *session.Session, awsConfig *aws.Config) (string, error) {
        stsSvc := sts.New(sess, awsConfig)
        req, err := stsSvc.GetCallerIdentity(&sts.GetCallerIdentityInput{})
        if err != nil {
                return "", err
        }
        return *req.Account, nil
}

func stringArrayContains(s []string, e string) bool {
        for _, a := range s {
                if a == e {
                        return true
                }
        }
        return false
}

func getLatestImageBuilderAmi(imageBuilderReaderRoleArn string, imagePipelineArn string, imagebuilderListImagePipelineImages func(*imagebuilder.ListImagePipelineImagesInput) (*imagebuilder.ListImagePipelineImagesOutput, error)) (string, error) {
        // the `imagebuilderListImagePipelineImages` parameter should not be provided in production. It allows
        // us to test this function by mocking `imagebuilder.ListImagePipelineImages` in the tests.
        if imagebuilderListImagePipelineImages == nil {
                sess := session.Must(session.NewSession(&aws.Config{
                        Region: aws.String("us-east-1"),
                }))
                creds := stscreds.NewCredentials(sess, imageBuilderReaderRoleArn)
                imageBuilderSvc := imagebuilder.New(sess, &aws.Config{Credentials: creds})
                imagebuilderListImagePipelineImages = imageBuilderSvc.ListImagePipelineImages
        }

        // get all images created by the image builder pipeline
        listImagePipelineImagesOutput, err := imagebuilderListImagePipelineImages(&imagebuilder.ListImagePipelineImagesInput{
                ImagePipelineArn: aws.String(imagePipelineArn),
        })
        if err != nil {
                Config.Logger.Printf("Error getting '%s' AMIs: %v", imagePipelineArn, err)
                return "", err
        }
        imagePipelineImages := listImagePipelineImagesOutput.ImageSummaryList

        // if the result is paginated, get the rest of the images
        for listImagePipelineImagesOutput.NextToken != nil {
                listImagePipelineImagesOutput, err = imagebuilderListImagePipelineImages(&imagebuilder.ListImagePipelineImagesInput{
                        ImagePipelineArn: aws.String(imagePipelineArn),
                        NextToken:        listImagePipelineImagesOutput.NextToken,
                })
                if err != nil {
                        Config.Logger.Printf("Error getting '%s' AMIs: %v", imagePipelineArn, err)
                        return "", err
                }
                imagePipelineImages = append(imagePipelineImages, listImagePipelineImagesOutput.ImageSummaryList...)
        }

        if len(imagePipelineImages) == 0 {
                return "", fmt.Errorf("no '%s' AMI found", imagePipelineArn)
        }

        // find the most recently created image
        latestImage := imagePipelineImages[0]
        latestTimeStamp, _ := time.Parse(time.RFC3339, *latestImage.DateCreated)
        if len(imagePipelineImages) > 1 {
                for _, image := range imagePipelineImages[1:] {
                        creationTimeStamp, _ := time.Parse(time.RFC3339, *image.DateCreated)
                        if creationTimeStamp.After(latestTimeStamp) {
                                latestTimeStamp = creationTimeStamp
                                latestImage = image
                        }
                }
        }

        ami := latestImage.OutputResources.Amis[0].Image
        Config.Logger.Printf("Using latest '%s' AMI '%s', created on %s", imagePipelineArn, *ami, latestTimeStamp)
        return *ami, nil
}

1	package hatchery
2
3	import (
4	"bytes"
5	"context"
6	"encoding/json"
7	"errors"
8	"fmt"
9	"net/http"
10	"os"
11	"regexp"
12	"strconv"
13	"strings"
14	"time"
15
16	"github.com/aws/aws-sdk-go/aws"
17	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
18	"github.com/aws/aws-sdk-go/aws/session"
19	"github.com/aws/aws-sdk-go/service/imagebuilder"
20	"github.com/aws/aws-sdk-go/service/sts"
21	)
22
23	type APIKeyStruct struct {
24	APIKey string `json:"api_key"`
25	KeyID string `json:"key_id"`
26	}
27
28	type WorkspaceKernelStatusStruct struct {
29	LastActivityTime string `json:"last_activity"`
30	}
31
32	func StrToInt(str string) (string, error) {	×
33	nonFractionalPart := strings.Split(str, ".")	×
34	return nonFractionalPart[0], nil	×
35	}	×
36
37	func mem(str string) (string, error) {	×
38	res := regexp.MustCompile(`(\d*)([M\|G])ib?`)	×
39	matches := res.FindStringSubmatch(str)	×
40	num, err := strconv.Atoi(matches[1])	×
41	if err != nil {	×
42	return "", err	×
43	}	×
44	if matches[2] == "G" {	×
45	num = num * 1024	×
46	}	×
47	return strconv.Itoa(num), nil	×
48	}
49
50	func cpu(str string) (string, error) {	×
51	num, err := strconv.Atoi(str[:strings.IndexByte(str, '.')])	×
52	if err != nil {	×
53	return "", err	×
54	}	×
55	num = num * 1024	×
56	return strconv.Itoa(num), nil	×
57	}
58
59	// Escapism escapes characters not allowed into hex with -
60	func escapism(input string) string {	5✔
61	safeBytes := "abcdefghijklmnopqrstuvwxyz0123456789"	5✔
62	var escaped string	5✔
63	for _, v := range input {	47✔
64	if !characterInString(v, safeBytes) {	45✔
65	hexCode := fmt.Sprintf("%2x", v)	3✔
66	escaped += "-" + hexCode	3✔
67	} else {	42✔
68	escaped += string(v)	39✔
69	}	39✔
70	}
71	return escaped	5✔
72	}
73
74	func characterInString(a rune, list string) bool {	42✔
75	for _, b := range list {	612✔
76	if b == a {	609✔
77	return true	39✔
78	}	39✔
79	}
80	return false	3✔
81	}
82
83	func truncateString(str string, num int) string {	×
84	bnoden := str	×
85	if len(str) > num {	×
86	bnoden = str[0:num]	×
87	}	×
88	if bnoden[len(bnoden)-1] == '-' {	×
89	bnoden = bnoden[0 : len(bnoden)-2]	×
90	}	×
91	return bnoden	×
92	}
93
94	// API key related helper functions
95	// Make http request with header and body
96	func MakeARequestWithContext(ctx context.Context, method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body bytes.Buffer) (http.Response, error) {	×
97	if headers == nil {	×
98	headers = make(map[string]string)	×
99	}	×
100	if accessToken != "" {	×
101	headers["Authorization"] = "Bearer " + accessToken	×
102	}	×
103	if contentType != "" {	×
104	headers["Content-Type"] = contentType	×
105	}	×
106	client := &http.Client{Timeout: 10 * time.Second}	×
107	var req *http.Request	×
108	var err error	×
109	if body == nil {	×
110	req, err = http.NewRequestWithContext(ctx, method, apiEndpoint, nil)	×
111	} else {	×
112	req, err = http.NewRequestWithContext(ctx, method, apiEndpoint, body)	×
113	}	×
114
115	if err != nil {	×
116	return nil, errors.New("Error occurred during generating HTTP request: " + err.Error())	×
117	}	×
118	for k, v := range headers {	×
119	req.Header.Add(k, v)	×
120	}	×
121	resp, err := client.Do(req)	×
122	if err != nil {	×
123	return nil, errors.New("Error occurred during making HTTP request: " + err.Error())	×
124	}	×
125	return resp, nil	×
126	}
127
128	func getFenceURL() string {	×
129	fenceURL := "http://fence-service/"	×
130	_, ok := os.LookupEnv("GEN3_ENDPOINT")	×
131	if ok {	×
132	fenceURL = "https://" + os.Getenv("GEN3_ENDPOINT") + "/user/"	×
133	}	×
134	return fenceURL	×
135	}
136
137	func getAmbassadorURL() string {	×
138	ambassadorURL := "http://ambassador-service/"	×
139	_, ok := os.LookupEnv("GEN3_ENDPOINT")	×
140	if ok {	×
141	ambassadorURL = "https://" + os.Getenv("GEN3_ENDPOINT") + "/lw-workspace/proxy/"	×
142	}	×
143	return ambassadorURL	×
144	}
145
146	func getAPIKeyWithContext(ctx context.Context, accessToken string) (apiKey *APIKeyStruct, err error) {	×
147	if accessToken == "" {	×
148	return nil, errors.New("No valid access token")	×
149	}	×
150
151	fenceAPIKeyURL := getFenceURL() + "credentials/api/"	×
152	body := bytes.NewBufferString("{\"scope\": [\"data\", \"user\"]}")	×
153		×
154	resp, err := MakeARequestWithContext(ctx, "POST", fenceAPIKeyURL, accessToken, "application/json", nil, body)	×
155	if err != nil {	×
156	return nil, err	×
157	}	×
158
159	if resp != nil && resp.StatusCode != 200 {	×
160	return nil, errors.New("Error occurred when creating API key with error code " + strconv.Itoa(resp.StatusCode))	×
161	}	×
162	defer resp.Body.Close()	×
163		×
164	fenceApiKeyResponse := new(APIKeyStruct)	×
165	err = json.NewDecoder(resp.Body).Decode(fenceApiKeyResponse)	×
166	if err != nil {	×
167	return nil, errors.New("Unable to decode API key response: " + err.Error())	×
168	}	×
169	return fenceApiKeyResponse, nil	×
170	}
171
172	func deleteAPIKeyWithContext(ctx context.Context, accessToken string, apiKeyID string) error {	×
173	if accessToken == "" {	×
174	return errors.New("No valid access token")	×
175	}	×
176
177	fenceDeleteAPIKeyURL := getFenceURL() + "credentials/api/" + apiKeyID	×
178	resp, err := MakeARequestWithContext(ctx, "DELETE", fenceDeleteAPIKeyURL, accessToken, "", nil, nil)	×
179	if err != nil {	×
180	return err	×
181	}	×
182	if resp != nil && resp.StatusCode != 204 {	×
183	return errors.New("Error occurred when deleting API key with error code " + strconv.Itoa(resp.StatusCode))	×
184	}	×
185	return nil	×
186	}
187
188	func getKernelIdleTimeWithContext(ctx context.Context, accessToken string) (lastActivityTime int64, err error) {	×
189	if accessToken == "" {	×
190	return -1, errors.New("No valid access token")	×
191	}	×
192
193	workspaceKernelStatusURL := getAmbassadorURL() + "api/status"	×
194	resp, err := MakeARequestWithContext(ctx, "GET", workspaceKernelStatusURL, accessToken, "", nil, nil)	×
195	if err != nil {	×
196	return -1, err	×
197	}	×
198	if resp != nil && resp.StatusCode != 200 {	×
199	return -1, errors.New("Error occurred when getting workspace kernel status with error code " + strconv.Itoa(resp.StatusCode))	×
200	}	×
201	defer resp.Body.Close()	×
202		×
203	workspaceKernelStatusResponse := new(WorkspaceKernelStatusStruct)	×
204	err = json.NewDecoder(resp.Body).Decode(workspaceKernelStatusResponse)	×
205	if err != nil {	×
206	return -1, errors.New("Unable to decode workspace kernel status response: " + err.Error())	×
207	}	×
208	lastAct, err := time.Parse(time.RFC3339, workspaceKernelStatusResponse.LastActivityTime)	×
209	if err != nil {	×
210	return -1, errors.New("Unable to parse last activity time: " + err.Error())	×
211	}	×
212	return lastAct.Unix() * 1000, nil	×
213	}
214
215	func getAwsAccountId(sess session.Session, awsConfig aws.Config) (string, error) {	4✔
216	stsSvc := sts.New(sess, awsConfig)	4✔
217	req, err := stsSvc.GetCallerIdentity(&sts.GetCallerIdentityInput{})	4✔
218	if err != nil {	8✔
219	return "", err	4✔
220	}	4✔
221	return *req.Account, nil	×
222	}
223
224	func stringArrayContains(s []string, e string) bool {	4✔
225	for _, a := range s {	9✔
226	if a == e {	7✔
227	return true	2✔
228	}	2✔
229	}
230	return false	2✔
231	}
232
233	func getLatestImageBuilderAmi(imageBuilderReaderRoleArn string, imagePipelineArn string, imagebuilderListImagePipelineImages func(imagebuilder.ListImagePipelineImagesInput) (imagebuilder.ListImagePipelineImagesOutput, error)) (string, error) {	1✔
234	// the `imagebuilderListImagePipelineImages` parameter should not be provided in production. It allows	1✔
235	// us to test this function by mocking `imagebuilder.ListImagePipelineImages` in the tests.	1✔
236	if imagebuilderListImagePipelineImages == nil {	1✔
NEW 237	sess := session.Must(session.NewSession(&aws.Config{	×
NEW 238	Region: aws.String("us-east-1"),	×
NEW 239	}))	×
NEW 240	creds := stscreds.NewCredentials(sess, imageBuilderReaderRoleArn)	×
NEW 241	imageBuilderSvc := imagebuilder.New(sess, &aws.Config{Credentials: creds})	×
NEW 242	imagebuilderListImagePipelineImages = imageBuilderSvc.ListImagePipelineImages	×
NEW 243	}	×
244
245	// get all images created by the image builder pipeline
246	listImagePipelineImagesOutput, err := imagebuilderListImagePipelineImages(&imagebuilder.ListImagePipelineImagesInput{	1✔
247	ImagePipelineArn: aws.String(imagePipelineArn),	1✔
248	})	1✔
249	if err != nil {	1✔
NEW 250	Config.Logger.Printf("Error getting '%s' AMIs: %v", imagePipelineArn, err)	×
NEW 251	return "", err	×
NEW 252	}	×
253	imagePipelineImages := listImagePipelineImagesOutput.ImageSummaryList	1✔
254		1✔
255	// if the result is paginated, get the rest of the images	1✔
256	for listImagePipelineImagesOutput.NextToken != nil {	2✔
257	listImagePipelineImagesOutput, err = imagebuilderListImagePipelineImages(&imagebuilder.ListImagePipelineImagesInput{	1✔
258	ImagePipelineArn: aws.String(imagePipelineArn),	1✔
259	NextToken: listImagePipelineImagesOutput.NextToken,	1✔
260	})	1✔
261	if err != nil {	1✔
NEW 262	Config.Logger.Printf("Error getting '%s' AMIs: %v", imagePipelineArn, err)	×
NEW 263	return "", err	×
NEW 264	}	×
265	imagePipelineImages = append(imagePipelineImages, listImagePipelineImagesOutput.ImageSummaryList...)	1✔
266	}
267
268	if len(imagePipelineImages) == 0 {	1✔
NEW 269	return "", fmt.Errorf("no '%s' AMI found", imagePipelineArn)	×
NEW 270	}	×
271
272	// find the most recently created image
273	latestImage := imagePipelineImages[0]	1✔
274	latestTimeStamp, _ := time.Parse(time.RFC3339, *latestImage.DateCreated)	1✔
275	if len(imagePipelineImages) > 1 {	2✔
276	for _, image := range imagePipelineImages[1:] {	2✔
277	creationTimeStamp, _ := time.Parse(time.RFC3339, *image.DateCreated)	1✔
278	if creationTimeStamp.After(latestTimeStamp) {	2✔
279	latestTimeStamp = creationTimeStamp	1✔
280	latestImage = image	1✔
281	}	1✔
282	}
283	}
284
285	ami := latestImage.OutputResources.Amis[0].Image	1✔
286	Config.Logger.Printf("Using latest '%s' AMI '%s', created on %s", imagePipelineArn, *ami, latestTimeStamp)	1✔
287	return *ami, nil	1✔
288	}

uc-cdis / hatchery / 8299721104

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous