13829663743

Committed 13 Mar 2025 07:55AM UTC coverage: 90.604% (+0.5%) from 90.121%

Build # 13829663743

Build Type

Pull #156

github

Committed by

thschmitt

Commit Message

Add network package with common http client

Moved common code for handling HTTP requests in a new shared
network package to avoid duplication and ensure that each plugin
handles network requests the same way.

The common HTTP client code takes care of a couple of cross-cutting
concerns needed by all plugins and the main HTTP executor:

- Logs request and response when debug flag is set. Added a new
  resettableReader which preserves the request and response bodies while
  they are being read and forwards them to the debug logger.

- Added retries for all HTTP requests. This also leverages the
  resettableReader to ensure the request body can be replayed.

- The `CommandBuilder` generates now an operation id which is set on every
  request the uipathcli performs. The same operation id is kept for the
  duration of the whole command execution which makes it easier to
  correlate multiple requests performed by a single command.

- The `HttpClient` also sets transport-related settings like certificate
  validation and response header timeout.

- Using the built-in context instead of a custom requestError channel.

Pull Request Pull Request #156: Add network package with common http client

Run Details

478 of 505 new or added lines in 26 files covered. (94.65%)

8 existing lines in 5 files now uncovered.

5371 of 5928 relevant lines covered (90.6%)

1.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.54

/plugin/digitizer/digitize_command.go

package digitzer

import (
        "bytes"
        "context"
        "encoding/json"
        "errors"
        "fmt"
        "io"
        "mime/multipart"
        "net/http"
        "net/textproto"
        "net/url"
        "strings"
        "time"

        "github.com/UiPath/uipathcli/log"
        "github.com/UiPath/uipathcli/output"
        "github.com/UiPath/uipathcli/plugin"
        "github.com/UiPath/uipathcli/utils/network"
        "github.com/UiPath/uipathcli/utils/stream"
        "github.com/UiPath/uipathcli/utils/visualization"
)

// The DigitizeCommand is a convenient wrapper over the async digitizer API
// to make it seem like it is a single sync call.
type DigitizeCommand struct{}

func (c DigitizeCommand) Command() plugin.Command {
        return *plugin.NewCommand("du").
                WithCategory("digitization", "Document Digitization", "Digitizes a document, extracting its Document Object Model (DOM) and text.").
                WithOperation("digitize", "Digitize file", "Digitize the given file").
                WithParameter("project-id", plugin.ParameterTypeString, "The project id", false).
                WithParameter("file", plugin.ParameterTypeBinary, "The file to digitize", true).
                WithParameter("content-type", plugin.ParameterTypeString, "The content type", false)
}

func (c DigitizeCommand) Execute(ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) error {
        if ctx.Organization == "" {
                return errors.New("Organization is not set")
        }
        if ctx.Tenant == "" {
                return errors.New("Tenant is not set")
        }
        documentId, err := c.startDigitization(ctx, logger)
        if err != nil {
                return err
        }

        for i := 1; i <= 60; i++ {
                finished, err := c.waitForDigitization(documentId, ctx, writer, logger)
                if err != nil {
                        return err
                }
                if finished {
                        return nil
                }
                time.Sleep(1 * time.Second)
        }
        return fmt.Errorf("Digitization with documentId '%s' did not finish in time", documentId)
}

func (c DigitizeCommand) startDigitization(ctx plugin.ExecutionContext, logger log.Logger) (string, error) {
        uploadBar := visualization.NewProgressBar(logger)
        defer uploadBar.Remove()
        context, cancel := context.WithCancelCause(context.Background())
        request := c.createDigitizeRequest(ctx, uploadBar, cancel)
        client := network.NewHttpClient(logger, c.httpClientSettings(ctx))
        response, err := client.SendWithContext(request, context)
        if err != nil {
                return "", err
        }
        defer response.Body.Close()
        body, err := io.ReadAll(response.Body)
        if err != nil {
                return "", fmt.Errorf("Error reading response: %w", err)
        }
        if response.StatusCode != http.StatusAccepted {
                return "", fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))
        }
        var result digitizeResponse
        err = json.Unmarshal(body, &result)
        if err != nil {
                return "", fmt.Errorf("Error parsing json response: %w", err)
        }
        return result.DocumentId, nil
}

func (c DigitizeCommand) waitForDigitization(documentId string, ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) (bool, error) {
        request := c.createDigitizeStatusRequest(documentId, ctx)
        client := network.NewHttpClient(logger, c.httpClientSettings(ctx))
        response, err := client.Send(request)
        if err != nil {
                return true, err
        }
        defer response.Body.Close()
        body, err := io.ReadAll(response.Body)
        if err != nil {
                return true, fmt.Errorf("Error reading response: %w", err)
        }
        if response.StatusCode != http.StatusOK {
                return true, fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))
        }
        var result digitizeResultResponse
        err = json.Unmarshal(body, &result)
        if err != nil {
                return true, fmt.Errorf("Error parsing json response: %w", err)
        }
        if result.Status == "NotStarted" || result.Status == "Running" {
                return false, nil
        }
        err = writer.WriteResponse(*output.NewResponseInfo(response.StatusCode, response.Status, response.Proto, response.Header, bytes.NewReader(body)))
        return true, err
}

func (c DigitizeCommand) createDigitizeRequest(ctx plugin.ExecutionContext, uploadBar *visualization.ProgressBar, cancel context.CancelCauseFunc) *network.HttpRequest {
        projectId := c.getProjectId(ctx.Parameters)

        file := ctx.Input
        if file == nil {
                file = c.getFileParameter(ctx.Parameters)
        }
        contentType := c.getParameter("content-type", ctx.Parameters)
        if contentType == "" {
                contentType = "application/octet-stream"
        }

        streamSize, _ := file.Size()
        bodyReader, bodyWriter := io.Pipe()
        formDataContentType := c.writeMultipartBody(bodyWriter, file, contentType, cancel)
        uploadReader := c.progressReader("uploading...", "completing  ", bodyReader, streamSize, uploadBar)

        uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + "/digitization/start?api-version=1"
        header := http.Header{
                "Content-Type": {formDataContentType},
        }
        for key, value := range ctx.Auth.Header {
                header.Set(key, value)
        }
        return network.NewHttpPostRequest(uri, header, uploadReader, -1)
}

func (c DigitizeCommand) progressReader(text string, completedText string, reader io.Reader, length int64, progressBar *visualization.ProgressBar) io.Reader {
        if length < 10*1024*1024 {
                return reader
        }
        return visualization.NewProgressReader(reader, func(progress visualization.Progress) {
                displayText := text
                if progress.Completed {
                        displayText = completedText
                }
                progressBar.UpdateProgress(displayText, progress.BytesRead, length, progress.BytesPerSecond)
        })
}

func (c DigitizeCommand) formatUri(baseUri url.URL, org string, tenant string, projectId string) string {
        path := baseUri.Path
        if baseUri.Path == "" {
                path = "/{organization}/{tenant}/du_/api/framework/projects/{projectId}"
        }
        path = strings.ReplaceAll(path, "{organization}", org)
        path = strings.ReplaceAll(path, "{tenant}", tenant)
        path = strings.ReplaceAll(path, "{projectId}", projectId)
        path = strings.TrimSuffix(path, "/")
        return fmt.Sprintf("%s://%s%s", baseUri.Scheme, baseUri.Host, path)
}

func (c DigitizeCommand) createDigitizeStatusRequest(documentId string, ctx plugin.ExecutionContext) *network.HttpRequest {
        projectId := c.getProjectId(ctx.Parameters)
        uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + fmt.Sprintf("/digitization/result/%s?api-version=1", documentId)
        header := http.Header{}
        for key, value := range ctx.Auth.Header {
                header.Set(key, value)
        }
        return network.NewHttpGetRequest(uri, header)
}

func (c DigitizeCommand) writeMultipartForm(writer *multipart.Writer, stream stream.Stream, contentType string) error {
        filePart := textproto.MIMEHeader{}
        filePart.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, stream.Name()))
        filePart.Set("Content-Type", contentType)
        w, err := writer.CreatePart(filePart)
        if err != nil {
                return fmt.Errorf("Error creating form field 'file': %w", err)
        }
        data, err := stream.Data()
        if err != nil {
                return err
        }
        defer data.Close()
        _, err = io.Copy(w, data)
        if err != nil {
                return fmt.Errorf("Error writing form field 'file': %w", err)
        }
        return nil
}

func (c DigitizeCommand) writeMultipartBody(bodyWriter *io.PipeWriter, stream stream.Stream, contentType string, cancel context.CancelCauseFunc) string {
        formWriter := multipart.NewWriter(bodyWriter)
        go func() {
                defer bodyWriter.Close()
                defer formWriter.Close()
                err := c.writeMultipartForm(formWriter, stream, contentType)
                if err != nil {
                        cancel(err)
                        return
                }
        }()
        return formWriter.FormDataContentType()
}

func (c DigitizeCommand) getProjectId(parameters []plugin.ExecutionParameter) string {
        projectId := c.getParameter("project-id", parameters)
        if projectId == "" {
                projectId = "00000000-0000-0000-0000-000000000000"
        }
        return projectId
}

func (c DigitizeCommand) getParameter(name string, parameters []plugin.ExecutionParameter) string {
        result := ""
        for _, p := range parameters {
                if p.Name == name {
                        if data, ok := p.Value.(string); ok {
                                result = data
                                break
                        }
                }
        }
        return result
}

func (c DigitizeCommand) getFileParameter(parameters []plugin.ExecutionParameter) stream.Stream {
        var result stream.Stream
        for _, p := range parameters {
                if p.Name == "file" {
                        if stream, ok := p.Value.(stream.Stream); ok {
                                result = stream
                                break
                        }
                }
        }
        return result
}

func (c DigitizeCommand) httpClientSettings(ctx plugin.ExecutionContext) network.HttpClientSettings {
        return *network.NewHttpClientSettings(
                ctx.Debug,
                ctx.Settings.OperationId,
                ctx.Settings.Timeout,
                ctx.Settings.MaxAttempts,
                ctx.Settings.Insecure)
}

func NewDigitizeCommand() *DigitizeCommand {
        return &DigitizeCommand{}
}

1	package digitzer
2
3	import (
4	"bytes"
5	"context"
6	"encoding/json"
7	"errors"
8	"fmt"
9	"io"
10	"mime/multipart"
11	"net/http"
12	"net/textproto"
13	"net/url"
14	"strings"
15	"time"
16
17	"github.com/UiPath/uipathcli/log"
18	"github.com/UiPath/uipathcli/output"
19	"github.com/UiPath/uipathcli/plugin"
20	"github.com/UiPath/uipathcli/utils/network"
21	"github.com/UiPath/uipathcli/utils/stream"
22	"github.com/UiPath/uipathcli/utils/visualization"
23	)
24
25	// The DigitizeCommand is a convenient wrapper over the async digitizer API
26	// to make it seem like it is a single sync call.
27	type DigitizeCommand struct{}
28
29	func (c DigitizeCommand) Command() plugin.Command {	1✔
30	return *plugin.NewCommand("du").	1✔
31	WithCategory("digitization", "Document Digitization", "Digitizes a document, extracting its Document Object Model (DOM) and text.").	1✔
32	WithOperation("digitize", "Digitize file", "Digitize the given file").	1✔
33	WithParameter("project-id", plugin.ParameterTypeString, "The project id", false).	1✔
34	WithParameter("file", plugin.ParameterTypeBinary, "The file to digitize", true).	1✔
35	WithParameter("content-type", plugin.ParameterTypeString, "The content type", false)	1✔
36	}	1✔
37
38	func (c DigitizeCommand) Execute(ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) error {	1✔
39	if ctx.Organization == "" {	2✔
40	return errors.New("Organization is not set")	1✔
41	}	1✔
42	if ctx.Tenant == "" {	2✔
43	return errors.New("Tenant is not set")	1✔
44	}	1✔
45	documentId, err := c.startDigitization(ctx, logger)	1✔
46	if err != nil {	2✔
47	return err	1✔
48	}	1✔
49
50	for i := 1; i <= 60; i++ {	2✔
51	finished, err := c.waitForDigitization(documentId, ctx, writer, logger)	1✔
52	if err != nil {	2✔
53	return err	1✔
54	}	1✔
55	if finished {	2✔
56	return nil	1✔
57	}	1✔
58	time.Sleep(1 * time.Second)	×
59	}
60	return fmt.Errorf("Digitization with documentId '%s' did not finish in time", documentId)	×
61	}
62
63	func (c DigitizeCommand) startDigitization(ctx plugin.ExecutionContext, logger log.Logger) (string, error) {	1✔
64	uploadBar := visualization.NewProgressBar(logger)	1✔
65	defer uploadBar.Remove()	1✔
66	context, cancel := context.WithCancelCause(context.Background())	1✔
67	request := c.createDigitizeRequest(ctx, uploadBar, cancel)	1✔
68	client := network.NewHttpClient(logger, c.httpClientSettings(ctx))	1✔
69	response, err := client.SendWithContext(request, context)	1✔
70	if err != nil {	2✔
71	return "", err	1✔
72	}	1✔
73	defer response.Body.Close()	1✔
74	body, err := io.ReadAll(response.Body)	1✔
75	if err != nil {	1✔
76	return "", fmt.Errorf("Error reading response: %w", err)	×
77	}	×
78	if response.StatusCode != http.StatusAccepted {	2✔
79	return "", fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))	1✔
80	}	1✔
81	var result digitizeResponse	1✔
82	err = json.Unmarshal(body, &result)	1✔
83	if err != nil {	1✔
84	return "", fmt.Errorf("Error parsing json response: %w", err)	×
85	}	×
86	return result.DocumentId, nil	1✔
87	}
88
89	func (c DigitizeCommand) waitForDigitization(documentId string, ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) (bool, error) {	1✔
90	request := c.createDigitizeStatusRequest(documentId, ctx)	1✔
91	client := network.NewHttpClient(logger, c.httpClientSettings(ctx))	1✔
92	response, err := client.Send(request)	1✔
93	if err != nil {	1✔
94	return true, err	×
95	}	×
96	defer response.Body.Close()	1✔
97	body, err := io.ReadAll(response.Body)	1✔
98	if err != nil {	1✔
99	return true, fmt.Errorf("Error reading response: %w", err)	×
100	}	×
101	if response.StatusCode != http.StatusOK {	2✔
102	return true, fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))	1✔
103	}	1✔
104	var result digitizeResultResponse	1✔
105	err = json.Unmarshal(body, &result)	1✔
106	if err != nil {	1✔
107	return true, fmt.Errorf("Error parsing json response: %w", err)	×
108	}	×
109	if result.Status == "NotStarted" \|\| result.Status == "Running" {	1✔
110	return false, nil	×
111	}	×
112	err = writer.WriteResponse(*output.NewResponseInfo(response.StatusCode, response.Status, response.Proto, response.Header, bytes.NewReader(body)))	1✔
113	return true, err	1✔
114	}
115
116	func (c DigitizeCommand) createDigitizeRequest(ctx plugin.ExecutionContext, uploadBar visualization.ProgressBar, cancel context.CancelCauseFunc) network.HttpRequest {	1✔
117	projectId := c.getProjectId(ctx.Parameters)	1✔
118		1✔
119	file := ctx.Input	1✔
120	if file == nil {	2✔
121	file = c.getFileParameter(ctx.Parameters)	1✔
122	}	1✔
123	contentType := c.getParameter("content-type", ctx.Parameters)	1✔
124	if contentType == "" {	2✔
125	contentType = "application/octet-stream"	1✔
126	}	1✔
127
128	streamSize, _ := file.Size()	1✔
129	bodyReader, bodyWriter := io.Pipe()	1✔
130	formDataContentType := c.writeMultipartBody(bodyWriter, file, contentType, cancel)	1✔
131	uploadReader := c.progressReader("uploading...", "completing ", bodyReader, streamSize, uploadBar)	1✔
132		1✔
133	uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + "/digitization/start?api-version=1"	1✔
134	header := http.Header{	1✔
135	"Content-Type": {formDataContentType},	1✔
136	}	1✔
137	for key, value := range ctx.Auth.Header {	1✔
NEW 138	header.Set(key, value)	×
UNCOV 139	}	×
140	return network.NewHttpPostRequest(uri, header, uploadReader, -1)	1✔
141	}
142
143	func (c DigitizeCommand) progressReader(text string, completedText string, reader io.Reader, length int64, progressBar *visualization.ProgressBar) io.Reader {	1✔
144	if length < 1010241024 {	2✔
145	return reader	1✔
146	}	1✔
147	return visualization.NewProgressReader(reader, func(progress visualization.Progress) {	2✔
148	displayText := text	1✔
149	if progress.Completed {	2✔
150	displayText = completedText	1✔
151	}	1✔
152	progressBar.UpdateProgress(displayText, progress.BytesRead, length, progress.BytesPerSecond)	1✔
153	})
154	}
155
156	func (c DigitizeCommand) formatUri(baseUri url.URL, org string, tenant string, projectId string) string {	1✔
157	path := baseUri.Path	1✔
158	if baseUri.Path == "" {	2✔
159	path = "/{organization}/{tenant}/du_/api/framework/projects/{projectId}"	1✔
160	}	1✔
161	path = strings.ReplaceAll(path, "{organization}", org)	1✔
162	path = strings.ReplaceAll(path, "{tenant}", tenant)	1✔
163	path = strings.ReplaceAll(path, "{projectId}", projectId)	1✔
164	path = strings.TrimSuffix(path, "/")	1✔
165	return fmt.Sprintf("%s://%s%s", baseUri.Scheme, baseUri.Host, path)	1✔
166	}
167
168	func (c DigitizeCommand) createDigitizeStatusRequest(documentId string, ctx plugin.ExecutionContext) *network.HttpRequest {	1✔
169	projectId := c.getProjectId(ctx.Parameters)	1✔
170	uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + fmt.Sprintf("/digitization/result/%s?api-version=1", documentId)	1✔
171	header := http.Header{}	1✔
172	for key, value := range ctx.Auth.Header {	1✔
NEW 173	header.Set(key, value)	×
NEW 174	}	×
175	return network.NewHttpGetRequest(uri, header)	1✔
176	}
177
178	func (c DigitizeCommand) writeMultipartForm(writer *multipart.Writer, stream stream.Stream, contentType string) error {	1✔
179	filePart := textproto.MIMEHeader{}	1✔
180	filePart.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, stream.Name()))	1✔
181	filePart.Set("Content-Type", contentType)	1✔
182	w, err := writer.CreatePart(filePart)	1✔
183	if err != nil {	1✔
184	return fmt.Errorf("Error creating form field 'file': %w", err)	×
185	}	×
186	data, err := stream.Data()	1✔
187	if err != nil {	2✔
188	return err	1✔
189	}	1✔
190	defer data.Close()	1✔
191	_, err = io.Copy(w, data)	1✔
192	if err != nil {	1✔
193	return fmt.Errorf("Error writing form field 'file': %w", err)	×
194	}	×
195	return nil	1✔
196	}
197
198	func (c DigitizeCommand) writeMultipartBody(bodyWriter *io.PipeWriter, stream stream.Stream, contentType string, cancel context.CancelCauseFunc) string {	1✔
199	formWriter := multipart.NewWriter(bodyWriter)	1✔
200	go func() {	2✔
201	defer bodyWriter.Close()	1✔
202	defer formWriter.Close()	1✔
203	err := c.writeMultipartForm(formWriter, stream, contentType)	1✔
204	if err != nil {	2✔
205	cancel(err)	1✔
206	return	1✔
207	}	1✔
208	}()
209	return formWriter.FormDataContentType()	1✔
210	}
211
212	func (c DigitizeCommand) getProjectId(parameters []plugin.ExecutionParameter) string {	1✔
213	projectId := c.getParameter("project-id", parameters)	1✔
214	if projectId == "" {	2✔
215	projectId = "00000000-0000-0000-0000-000000000000"	1✔
216	}	1✔
217	return projectId	1✔
218	}
219
220	func (c DigitizeCommand) getParameter(name string, parameters []plugin.ExecutionParameter) string {	1✔
221	result := ""	1✔
222	for _, p := range parameters {	2✔
223	if p.Name == name {	2✔
224	if data, ok := p.Value.(string); ok {	2✔
225	result = data	1✔
226	break	1✔
227	}
228	}
229	}
230	return result	1✔
231	}
232
233	func (c DigitizeCommand) getFileParameter(parameters []plugin.ExecutionParameter) stream.Stream {	1✔
234	var result stream.Stream	1✔
235	for _, p := range parameters {	2✔
236	if p.Name == "file" {	2✔
237	if stream, ok := p.Value.(stream.Stream); ok {	2✔
238	result = stream	1✔
239	break	1✔
240	}
241	}
242	}
243	return result	1✔
244	}
245
246	func (c DigitizeCommand) httpClientSettings(ctx plugin.ExecutionContext) network.HttpClientSettings {	1✔
247	return *network.NewHttpClientSettings(	1✔
248	ctx.Debug,	1✔
249	ctx.Settings.OperationId,	1✔
250	ctx.Settings.Timeout,	1✔
251	ctx.Settings.MaxAttempts,	1✔
252	ctx.Settings.Insecure)	1✔
253	}	1✔
254
255	func NewDigitizeCommand() *DigitizeCommand {	1✔
256	return &DigitizeCommand{}	1✔
257	}	1✔

UiPath / uipathcli / 13829663743

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous