3673539607

Committed 12 Dec 2022 07:26AM UTC coverage: 74.396% (+0.4%) from 73.973%

Build Type

github

Committed by Suhas Karanth

Commit Message

feat: add HTTP extractor

Pull Request Pull Request #442: feat: add HTTP extractor

Run Details

245 of 245 new or added lines in 5 files covered. (100.0%)

5393 of 7249 relevant lines covered (74.4%)

0.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.88

/plugins/extractors/http/execute_request.go

package http

import (
        "bytes"
        "context"
        "encoding/json"
        "fmt"
        "io"
        "net/http"
)

func (e *Extractor) executeRequest(ctx context.Context) (interface{}, error) {
        cfg := e.config

        ctx, cancel := context.WithTimeout(ctx, cfg.Request.Timeout)
        defer cancel()

        req, err := buildRequest(ctx, cfg)
        if err != nil {
                return nil, err
        }

        resp, err := e.http.Do(req)
        defer drainBody(resp)
        if err != nil {
                return nil, fmt.Errorf("do request: %w", err)
        }

        return handleResponse(cfg, resp)
}

func buildRequest(ctx context.Context, cfg Config) (*http.Request, error) {
        reqCfg := cfg.Request

        body, err := asReader(reqCfg.Body)
        if err != nil {
                return nil, fmt.Errorf("encode request body: %w", err)
        }

        req, err := http.NewRequestWithContext(ctx, reqCfg.Method, reqCfg.URL, body)
        if err != nil {
                return nil, fmt.Errorf("create HTTP request: %w", err)
        }

        addQueryParams(req, reqCfg.QueryParams)

        for name, v := range reqCfg.Headers {
                req.Header.Set(name, v)
        }
        if req.Body != nil && req.Body != http.NoBody {
                req.Header.Set("Content-Type", "application/json")
        }
        req.Header.Set("Accept", "application/json")

        return req, nil
}

func addQueryParams(req *http.Request, params []QueryParam) {
        if len(params) == 0 {
                return
        }

        q := req.URL.Query()
        // First delete any possible conflicts. Cannot be done in a single loop
        // because params can have multiple entries with the same key.
        for _, p := range params {
                q.Del(p.Key)
        }
        for _, p := range params {
                q.Add(p.Key, p.Value)
        }
        req.URL.RawQuery = q.Encode()
}

func handleResponse(cfg Config, resp *http.Response) (interface{}, error) {
        if !has(cfg.SuccessCodes, resp.StatusCode) {
                return nil, fmt.Errorf("unsuccessful request: response status code: %d", resp.StatusCode)
        }

        var res interface{}
        if err := json.NewDecoder(resp.Body).Decode(&res); err != nil {
                return nil, fmt.Errorf("decode response: %w", err)
        }

        return res, nil
}

func asReader(v interface{}) (io.Reader, error) {
        if v == nil {
                return nil, nil
        }

        if body, ok := v.(string); ok {
                return bytes.NewBufferString(body), nil
        }

        var buf bytes.Buffer
        if err := json.NewEncoder(&buf).Encode(v); err != nil {
                return nil, err
        }

        return &buf, nil
}

// drainBody drains and closes the response body to avoid the following
// gotcha:
// http://devs.cloudimmunity.com/gotchas-and-common-mistakes-in-go-golang/index.html#close_http_resp_body
func drainBody(resp *http.Response) {
        if resp == nil {
                return
        }

        _, _ = io.Copy(io.Discard, resp.Body)
        _ = resp.Body.Close()
}

func has(haystack []int, needle int) bool {
        for _, n := range haystack {
                if n == needle {
                        return true
                }
        }

        return false
}

1	package http
2
3	import (
4	"bytes"
5	"context"
6	"encoding/json"
7	"fmt"
8	"io"
9	"net/http"
10	)
11
12	func (e *Extractor) executeRequest(ctx context.Context) (interface{}, error) {	1✔
13	cfg := e.config	1✔
14		1✔
15	ctx, cancel := context.WithTimeout(ctx, cfg.Request.Timeout)	1✔
16	defer cancel()	1✔
17		1✔
18	req, err := buildRequest(ctx, cfg)	1✔
19	if err != nil {	1✔
20	return nil, err	×
21	}	×
22
23	resp, err := e.http.Do(req)	1✔
24	defer drainBody(resp)	1✔
25	if err != nil {	2✔
26	return nil, fmt.Errorf("do request: %w", err)	1✔
27	}	1✔
28
29	return handleResponse(cfg, resp)	1✔
30	}
31
32	func buildRequest(ctx context.Context, cfg Config) (*http.Request, error) {	1✔
33	reqCfg := cfg.Request	1✔
34		1✔
35	body, err := asReader(reqCfg.Body)	1✔
36	if err != nil {	1✔
37	return nil, fmt.Errorf("encode request body: %w", err)	×
38	}	×
39
40	req, err := http.NewRequestWithContext(ctx, reqCfg.Method, reqCfg.URL, body)	1✔
41	if err != nil {	1✔
42	return nil, fmt.Errorf("create HTTP request: %w", err)	×
43	}	×
44
45	addQueryParams(req, reqCfg.QueryParams)	1✔
46		1✔
47	for name, v := range reqCfg.Headers {	2✔
48	req.Header.Set(name, v)	1✔
49	}	1✔
50	if req.Body != nil && req.Body != http.NoBody {	2✔
51	req.Header.Set("Content-Type", "application/json")	1✔
52	}	1✔
53	req.Header.Set("Accept", "application/json")	1✔
54		1✔
55	return req, nil	1✔
56	}
57
58	func addQueryParams(req *http.Request, params []QueryParam) {	1✔
59	if len(params) == 0 {	2✔
60	return	1✔
61	}	1✔
62
63	q := req.URL.Query()	1✔
64	// First delete any possible conflicts. Cannot be done in a single loop	1✔
65	// because params can have multiple entries with the same key.	1✔
66	for _, p := range params {	2✔
67	q.Del(p.Key)	1✔
68	}	1✔
69	for _, p := range params {	2✔
70	q.Add(p.Key, p.Value)	1✔
71	}	1✔
72	req.URL.RawQuery = q.Encode()	1✔
73	}
74
75	func handleResponse(cfg Config, resp *http.Response) (interface{}, error) {	1✔
76	if !has(cfg.SuccessCodes, resp.StatusCode) {	2✔
77	return nil, fmt.Errorf("unsuccessful request: response status code: %d", resp.StatusCode)	1✔
78	}	1✔
79
80	var res interface{}	1✔
81	if err := json.NewDecoder(resp.Body).Decode(&res); err != nil {	1✔
82	return nil, fmt.Errorf("decode response: %w", err)	×
83	}	×
84
85	return res, nil	1✔
86	}
87
88	func asReader(v interface{}) (io.Reader, error) {	1✔
89	if v == nil {	2✔
90	return nil, nil	1✔
91	}	1✔
92
93	if body, ok := v.(string); ok {	1✔
94	return bytes.NewBufferString(body), nil	×
95	}	×
96
97	var buf bytes.Buffer	1✔
98	if err := json.NewEncoder(&buf).Encode(v); err != nil {	1✔
99	return nil, err	×
100	}	×
101
102	return &buf, nil	1✔
103	}
104
105	// drainBody drains and closes the response body to avoid the following
106	// gotcha:
107	// http://devs.cloudimmunity.com/gotchas-and-common-mistakes-in-go-golang/index.html#close_http_resp_body
108	func drainBody(resp *http.Response) {	1✔
109	if resp == nil {	2✔
110	return	1✔
111	}	1✔
112
113	_, _ = io.Copy(io.Discard, resp.Body)	1✔
114	_ = resp.Body.Close()	1✔
115	}
116
117	func has(haystack []int, needle int) bool {	1✔
118	for _, n := range haystack {	2✔
119	if n == needle {	2✔
120	return true	1✔
121	}	1✔
122	}
123
124	return false	1✔
125	}

odpf / meteor / 3673539607

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous