10882594019

Committed 16 Sep 2024 05:47AM UTC coverage: 80.604% (+0.1%) from 80.5%

Build # 10882594019

Build Type

push

github

Committed by

web-flow

Commit Message

Releases/v1.2.0 (#1236)

* chore: merge develop to releases/v1.2.0 (#1227)

* feat: Github Migration  (#1146)

* feat: migrate to github actions

* chore: migrate from circle ci

* Update develop.yml (#1148)

* Update develop.yml (#1149)

* Update develop.yml

* Update develop.yml

* Update develop.yml

* Fix go mod tidy

* Replaced curve instance elliptic.P256() with crypto.S256() (#1150)

* fix: Update Dockerfile with version bumps (#1152)

* chore: hotfix missing workflow

* chore: fix dockerfile versions

* Feature/v1.1.0 (#1154)

* v1.2.0 (#965)

* Hotfix-logImprovements (#952)

* Set up blocknumber and epoch in logs

* updated blocknumber and epoch logger info in every command

* Hotfix-getDataFromAPI (#951)

* Changed numm of retry attempts

* removed redundant retry attempts

* corrected tests

* changed http timeout and logged time elapsed to fetch data (#954)

* Updated version (#960)

* Updated version

* updated version to v1.2.0

* version update (#972)

* Merged `v1.3.0-alpha` into `v1.0.5` (#973)

* Merged `v1` into `v1.3.0-aplha` with hotfixes (#966)

* Hotfix-proposed data (#913)

* Updated propose data global variables correctly

* Fixed tests

* Returned correct waitForBlockCompletion error

* coverage increase

* GetLocalData returns type types.ProposeFileData

* fixed benchmark

* Fetched Last proposed from contracts (#917)

* fetched getLastProposedEpoch from contracts and tests for it

* typo fix

* V1 propose hotfix (#918)

* Change propose.go to get sorted proposed block ids.
* Fix sorted proposed block issue.

Signed-off-by: Ashish Kumar Mishra <ashish10677@gmail.com>

* allow stakers to addStake < minSafeRazor (#928)

* Call claimStakerReward only if there reward to claim (#926)

* Make contract call only if there is commission to claim

* Add tests for claimCommission file

* update check

* Hotfix-giveSorted (#921)

* ResetDi... (continued)

Run Details

887 of 1091 new or added lines in 40 files covered. (81.3%)

22 existing lines in 6 files now uncovered.

6641 of 8239 relevant lines covered (80.6%)

2524.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.33

/utils/api.go

package utils

import (
        "bytes"
        "encoding/json"
        "errors"
        "fmt"
        "io"
        "net/http"
        "razor/core"
        "regexp"
        "time"

        "razor/core/types"

        "github.com/PaesslerAG/jsonpath"
        "github.com/avast/retry-go"
        "github.com/gocolly/colly"
)

func GetDataFromAPI(commitParams *types.CommitParams, dataSourceURLStruct types.DataSourceURL) ([]byte, error) {
        cacheKey, err := generateCacheKey(dataSourceURLStruct.URL, dataSourceURLStruct.Body)
        if err != nil {
                log.Errorf("Error in generating cache key for API %s: %v", dataSourceURLStruct.URL, err)
                return nil, err
        }

        cachedData, found := commitParams.LocalCache.Read(cacheKey)
        if found {
                log.Debugf("Getting Data for URL %s from local cache...", dataSourceURLStruct.URL)
                return cachedData, nil
        }

        response, err := makeAPIRequest(commitParams.HttpClient, dataSourceURLStruct)
        if err != nil {
                return nil, err
        }

        // Storing the data into cache
        commitParams.LocalCache.Update(response, cacheKey, time.Now().Add(time.Second*time.Duration(core.StateLength)).Unix())
        return response, nil
}

func makeAPIRequest(httpClient *http.Client, dataSourceURLStruct types.DataSourceURL) ([]byte, error) {
        var requestBody io.Reader // Using the broader io.Reader interface here

        switch dataSourceURLStruct.Type {
        case "GET":
                // For HTTP GET requests, there is typically no request body.
                // So we explicitly set the requestBody to nil to indicate this absence.
                requestBody = nil
        case "POST":
                postBody, err := json.Marshal(dataSourceURLStruct.Body)
                if err != nil {
                        log.Errorf("Error in marshalling body of a POST request URL %s: %v", dataSourceURLStruct.URL, err)
                        return nil, err
                }
                requestBody = bytes.NewBuffer(postBody)
        default:
                return nil, errors.New("invalid request type")
        }

        var response []byte
        err := retry.Do(
                func() error {
                        responseBody, err := ProcessRequest(httpClient, dataSourceURLStruct, requestBody)
                        if err != nil {
                                log.Errorf("Error in processing %s request: %v", dataSourceURLStruct.Type, err)
                                return err
                        }
                        response = responseBody
                        return nil
                }, retry.Attempts(core.ProcessRequestRetryAttempts), retry.Delay(time.Second*time.Duration(core.ProcessRequestRetryDelay)))

        if err != nil {
                return nil, err
        }

        return response, nil
}

func parseJSONData(parsedJSON interface{}, selector string) (interface{}, error) {
        switch v := parsedJSON.(type) {
        case map[string]interface{}: // Handling JSON object response case
                return GetDataFromJSON(v, selector)

        case []interface{}: // Handling JSON array of objects response case
                if len(v) > 0 {
                        // The first element from JSON array is fetched
                        if elem, ok := v[0].(map[string]interface{}); ok {
                                return GetDataFromJSON(elem, selector)
                        }
                        log.Error("Element in array is not a JSON object")
                        return nil, errors.New("element in array is not a JSON object")
                }
                log.Error("Empty JSON array")
                return nil, errors.New("empty JSON array")
        default:
                log.Error("Unexpected JSON structure")
                return nil, errors.New("unexpected JSON structure")
        }
}

func GetDataFromJSON(jsonObject map[string]interface{}, selector string) (interface{}, error) {
        if selector[0] == '[' {
                selector = "$" + selector
        } else {
                selector = "$." + selector
        }
        return jsonpath.Get(selector, jsonObject)
}

func GetDataFromXHTML(dataSourceURLStruct types.DataSourceURL, selector string) (string, error) {
        c := colly.NewCollector()
        var priceData string
        c.OnXML(selector, func(e *colly.XMLElement) {
                priceData = e.Text
        })
        err := c.Visit(dataSourceURLStruct.URL)
        if err != nil {
                return "", err
        }
        return priceData, nil
}

func processHeaderValue(value string, re *regexp.Regexp) string {
        // check if any API authentication is required
        if re.MatchString(value) {
                return ReplaceValueWithDataFromENVFile(re, value)
        }
        return value
}

func addHeaderToRequest(request *http.Request, headerMap map[string]string) *http.Request {
        re := regexp.MustCompile(core.APIKeyRegex)
        for key, value := range headerMap {
                processedValue := processHeaderValue(value, re)
                log.Debugf("Adding key: %s, value: %s pair to header", key, value)
                request.Header.Add(key, processedValue)
        }
        return request
}

func ProcessRequest(httpClient *http.Client, dataSourceURLStruct types.DataSourceURL, requestBody io.Reader) ([]byte, error) {
        request, err := http.NewRequest(dataSourceURLStruct.Type, dataSourceURLStruct.URL, requestBody)
        if err != nil {
                return nil, err
        }
        requestWithHeader := addHeaderToRequest(request, dataSourceURLStruct.Header)
        response, err := httpClient.Do(requestWithHeader)
        if err != nil {
                log.Errorf("Error sending %s request URL %s: %v", dataSourceURLStruct.Type, dataSourceURLStruct.URL, err)
                return nil, err
        }
        defer response.Body.Close()
        // Success is indicated with 2xx status codes:
        statusOK := response.StatusCode >= 200 && response.StatusCode < 300
        if !statusOK {
                log.Errorf("API: %s responded with status code %d", dataSourceURLStruct.URL, response.StatusCode)
                return nil, fmt.Errorf("HTTP request failed with status code %d: %s", response.StatusCode, response.Status)
        }
        responseBody, err := io.ReadAll(response.Body)
        if err != nil {
                return nil, err
        }
        return responseBody, nil
}

1	package utils
2
3	import (
4	"bytes"
5	"encoding/json"
6	"errors"
7	"fmt"
8	"io"
9	"net/http"
10	"razor/core"
11	"regexp"
12	"time"
13
14	"razor/core/types"
15
16	"github.com/PaesslerAG/jsonpath"
17	"github.com/avast/retry-go"
18	"github.com/gocolly/colly"
19	)
20
21	func GetDataFromAPI(commitParams *types.CommitParams, dataSourceURLStruct types.DataSourceURL) ([]byte, error) {	31✔
22	cacheKey, err := generateCacheKey(dataSourceURLStruct.URL, dataSourceURLStruct.Body)	31✔
23	if err != nil {	33✔
24	log.Errorf("Error in generating cache key for API %s: %v", dataSourceURLStruct.URL, err)	2✔
25	return nil, err	2✔
26	}	2✔
27
28	cachedData, found := commitParams.LocalCache.Read(cacheKey)	29✔
29	if found {	29✔
UNCOV 30	log.Debugf("Getting Data for URL %s from local cache...", dataSourceURLStruct.URL)	×
UNCOV 31	return cachedData, nil	×
UNCOV 32	}	×
33
34	response, err := makeAPIRequest(commitParams.HttpClient, dataSourceURLStruct)	29✔
35	if err != nil {	35✔
36	return nil, err	6✔
37	}	6✔
38
39	// Storing the data into cache
40	commitParams.LocalCache.Update(response, cacheKey, time.Now().Add(time.Second*time.Duration(core.StateLength)).Unix())	23✔
41	return response, nil	23✔
42	}
43
44	func makeAPIRequest(httpClient *http.Client, dataSourceURLStruct types.DataSourceURL) ([]byte, error) {	29✔
45	var requestBody io.Reader // Using the broader io.Reader interface here	29✔
46		29✔
47	switch dataSourceURLStruct.Type {	29✔
48	case "GET":	21✔
49	// For HTTP GET requests, there is typically no request body.	21✔
50	// So we explicitly set the requestBody to nil to indicate this absence.	21✔
51	requestBody = nil	21✔
52	case "POST":	7✔
53	postBody, err := json.Marshal(dataSourceURLStruct.Body)	7✔
54	if err != nil {	7✔
55	log.Errorf("Error in marshalling body of a POST request URL %s: %v", dataSourceURLStruct.URL, err)	×
56	return nil, err	×
57	}	×
58	requestBody = bytes.NewBuffer(postBody)	7✔
59	default:	1✔
60	return nil, errors.New("invalid request type")	1✔
61	}
62
63	var response []byte	28✔
64	err := retry.Do(	28✔
65	func() error {	61✔
66	responseBody, err := ProcessRequest(httpClient, dataSourceURLStruct, requestBody)	33✔
67	if err != nil {	43✔
68	log.Errorf("Error in processing %s request: %v", dataSourceURLStruct.Type, err)	10✔
69	return err	10✔
70	}	10✔
71	response = responseBody	23✔
72	return nil	23✔
73	}, retry.Attempts(core.ProcessRequestRetryAttempts), retry.Delay(time.Second*time.Duration(core.ProcessRequestRetryDelay)))
74
75	if err != nil {	33✔
76	return nil, err	5✔
77	}	5✔
78
79	return response, nil	23✔
80	}
81
82	func parseJSONData(parsedJSON interface{}, selector string) (interface{}, error) {	24✔
83	switch v := parsedJSON.(type) {	24✔
84	case map[string]interface{}: // Handling JSON object response case	20✔
85	return GetDataFromJSON(v, selector)	20✔
86
87	case []interface{}: // Handling JSON array of objects response case	3✔
88	if len(v) > 0 {	5✔
89	// The first element from JSON array is fetched	2✔
90	if elem, ok := v[0].(map[string]interface{}); ok {	3✔
91	return GetDataFromJSON(elem, selector)	1✔
92	}	1✔
93	log.Error("Element in array is not a JSON object")	1✔
94	return nil, errors.New("element in array is not a JSON object")	1✔
95	}
96	log.Error("Empty JSON array")	1✔
97	return nil, errors.New("empty JSON array")	1✔
98	default:	1✔
99	log.Error("Unexpected JSON structure")	1✔
100	return nil, errors.New("unexpected JSON structure")	1✔
101	}
102	}
103
104	func GetDataFromJSON(jsonObject map[string]interface{}, selector string) (interface{}, error) {	25✔
105	if selector[0] == '[' {	26✔
106	selector = "$" + selector	1✔
107	} else {	25✔
108	selector = "$." + selector	24✔
109	}	24✔
110	return jsonpath.Get(selector, jsonObject)	25✔
111	}
112
113	func GetDataFromXHTML(dataSourceURLStruct types.DataSourceURL, selector string) (string, error) {	3✔
114	c := colly.NewCollector()	3✔
115	var priceData string	3✔
116	c.OnXML(selector, func(e *colly.XMLElement) {	4✔
117	priceData = e.Text	1✔
118	})	1✔
119	err := c.Visit(dataSourceURLStruct.URL)	3✔
120	if err != nil {	5✔
121	return "", err	2✔
122	}	2✔
123	return priceData, nil	1✔
124	}
125
126	func processHeaderValue(value string, re *regexp.Regexp) string {	8✔
127	// check if any API authentication is required	8✔
128	if re.MatchString(value) {	9✔
129	return ReplaceValueWithDataFromENVFile(re, value)	1✔
130	}	1✔
131	return value	7✔
132	}
133
134	func addHeaderToRequest(request http.Request, headerMap map[string]string) http.Request {	31✔
135	re := regexp.MustCompile(core.APIKeyRegex)	31✔
136	for key, value := range headerMap {	39✔
137	processedValue := processHeaderValue(value, re)	8✔
138	log.Debugf("Adding key: %s, value: %s pair to header", key, value)	8✔
139	request.Header.Add(key, processedValue)	8✔
140	}	8✔
141	return request	31✔
142	}
143
144	func ProcessRequest(httpClient *http.Client, dataSourceURLStruct types.DataSourceURL, requestBody io.Reader) ([]byte, error) {	33✔
145	request, err := http.NewRequest(dataSourceURLStruct.Type, dataSourceURLStruct.URL, requestBody)	33✔
146	if err != nil {	35✔
147	return nil, err	2✔
148	}	2✔
149	requestWithHeader := addHeaderToRequest(request, dataSourceURLStruct.Header)	31✔
150	response, err := httpClient.Do(requestWithHeader)	31✔
151	if err != nil {	33✔
152	log.Errorf("Error sending %s request URL %s: %v", dataSourceURLStruct.Type, dataSourceURLStruct.URL, err)	2✔
153	return nil, err	2✔
154	}	2✔
155	defer response.Body.Close()	29✔
156	// Success is indicated with 2xx status codes:	29✔
157	statusOK := response.StatusCode >= 200 && response.StatusCode < 300	29✔
158	if !statusOK {	35✔
159	log.Errorf("API: %s responded with status code %d", dataSourceURLStruct.URL, response.StatusCode)	6✔
160	return nil, fmt.Errorf("HTTP request failed with status code %d: %s", response.StatusCode, response.Status)	6✔
161	}	6✔
162	responseBody, err := io.ReadAll(response.Body)	23✔
163	if err != nil {	23✔
164	return nil, err	×
165	}	×
166	return responseBody, nil	23✔
167	}

razor-network / oracle-node / 10882594019

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous