13790794198

Committed 11 Mar 2025 02:38PM UTC coverage: 90.575% (+0.5%) from 90.121%

Build # 13790794198

Build Type

push

github

Committed by

thschmitt

Commit Message

Add network package with common http client

Moved common code for handling HTTP requests in a new shared
network package to avoid duplication and ensure that each plugin
handles network requests the same way.

The common HTTP client code takes care of a couple of cross-cutting
concerns needed by all plugins and the main HTTP executor:

- Logs request and response when debug flag is set. Added a new
  resettableReader which preserves the request and response bodies while
  they are being read and forwards them to the debug logger.

- Added retries for all HTTP requests. This also leverages the
  resettableReader to ensure the request body can be replayed.

- The `CommandBuilder` generates now an operation id which is set on every
  request the uipathcli performs. The same operation id is kept for the
  duration of the whole command execution which makes it easier to
  correlate multiple requests performed by a single command.

- The `HttpClient` also sets transport-related settings like certificate
  validation and response header timeout.

- Using the built-in context instead of a custom requestError channel.

Run Details

466 of 493 new or added lines in 26 files covered. (94.52%)

10 existing lines in 6 files now uncovered.

5372 of 5931 relevant lines covered (90.57%)

1.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.78

/plugin/digitizer/digitize_command.go

package digitzer

import (
        "bytes"
        "context"
        "encoding/json"
        "errors"
        "fmt"
        "io"
        "mime/multipart"
        "net/http"
        "net/textproto"
        "net/url"
        "strings"
        "time"

        "github.com/UiPath/uipathcli/log"
        "github.com/UiPath/uipathcli/output"
        "github.com/UiPath/uipathcli/plugin"
        "github.com/UiPath/uipathcli/utils/network"
        "github.com/UiPath/uipathcli/utils/stream"
        "github.com/UiPath/uipathcli/utils/visualization"
)

// The DigitizeCommand is a convenient wrapper over the async digitizer API
// to make it seem like it is a single sync call.
type DigitizeCommand struct{}

func (c DigitizeCommand) Command() plugin.Command {
        return *plugin.NewCommand("du").
                WithCategory("digitization", "Document Digitization", "Digitizes a document, extracting its Document Object Model (DOM) and text.").
                WithOperation("digitize", "Digitize file", "Digitize the given file").
                WithParameter("project-id", plugin.ParameterTypeString, "The project id", false).
                WithParameter("file", plugin.ParameterTypeBinary, "The file to digitize", true).
                WithParameter("content-type", plugin.ParameterTypeString, "The content type", false)
}

func (c DigitizeCommand) Execute(ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) error {
        if ctx.Organization == "" {
                return errors.New("Organization is not set")
        }
        if ctx.Tenant == "" {
                return errors.New("Tenant is not set")
        }
        documentId, err := c.startDigitization(ctx, logger)
        if err != nil {
                return err
        }

        for i := 1; i <= 60; i++ {
                finished, err := c.waitForDigitization(documentId, ctx, writer, logger)
                if err != nil {
                        return err
                }
                if finished {
                        return nil
                }
                time.Sleep(1 * time.Second)
        }
        return fmt.Errorf("Digitization with documentId '%s' did not finish in time", documentId)
}

func (c DigitizeCommand) startDigitization(ctx plugin.ExecutionContext, logger log.Logger) (string, error) {
        uploadBar := visualization.NewProgressBar(logger)
        defer uploadBar.Remove()
        context, cancel := context.WithCancelCause(context.Background())
        request := c.createDigitizeRequest(ctx, uploadBar, cancel)
        client := network.NewHttpClient(logger, c.httpClientSettings(ctx))
        response, err := client.SendWithContext(request, context)
        if err != nil {
                return "", err
        }
        defer response.Body.Close()
        body, err := io.ReadAll(response.Body)
        if err != nil {
                return "", fmt.Errorf("Error reading response: %w", err)
        }
        if response.StatusCode != http.StatusAccepted {
                return "", fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))
        }
        var result digitizeResponse
        err = json.Unmarshal(body, &result)
        if err != nil {
                return "", fmt.Errorf("Error parsing json response: %w", err)
        }
        return result.DocumentId, nil
}

func (c DigitizeCommand) waitForDigitization(documentId string, ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) (bool, error) {
        request := c.createDigitizeStatusRequest(documentId, ctx)
        client := network.NewHttpClient(logger, c.httpClientSettings(ctx))
        response, err := client.Send(request)
        if err != nil {
                return true, err
        }
        defer response.Body.Close()
        body, err := io.ReadAll(response.Body)
        if err != nil {
                return true, fmt.Errorf("Error reading response: %w", err)
        }
        if response.StatusCode != http.StatusOK {
                return true, fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))
        }
        var result digitizeResultResponse
        err = json.Unmarshal(body, &result)
        if err != nil {
                return true, fmt.Errorf("Error parsing json response: %w", err)
        }
        if result.Status == "NotStarted" || result.Status == "Running" {
                return false, nil
        }
        err = writer.WriteResponse(*output.NewResponseInfo(response.StatusCode, response.Status, response.Proto, response.Header, bytes.NewReader(body)))
        return true, err
}

func (c DigitizeCommand) createDigitizeRequest(ctx plugin.ExecutionContext, uploadBar *visualization.ProgressBar, cancel context.CancelCauseFunc) *network.HttpRequest {
        projectId := c.getProjectId(ctx.Parameters)

        file := ctx.Input
        if file == nil {
                file = c.getFileParameter(ctx.Parameters)
        }
        contentType := c.getParameter("content-type", ctx.Parameters)
        if contentType == "" {
                contentType = "application/octet-stream"
        }

        bodyReader, bodyWriter := io.Pipe()
        contentType, contentLength := c.writeMultipartBody(bodyWriter, file, contentType, cancel)
        uploadReader := c.progressReader("uploading...", "completing  ", bodyReader, contentLength, uploadBar)

        uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + "/digitization/start?api-version=1"
        header := http.Header{
                "Content-Type": {contentType},
        }
        for key, value := range ctx.Auth.Header {
                header.Set(key, value)
        }
        return network.NewHttpRequest(http.MethodPost, uri, header, uploadReader)
}

func (c DigitizeCommand) progressReader(text string, completedText string, reader io.Reader, length int64, progressBar *visualization.ProgressBar) io.Reader {
        if length < 10*1024*1024 {
                return reader
        }
        return visualization.NewProgressReader(reader, func(progress visualization.Progress) {
                displayText := text
                if progress.Completed {
                        displayText = completedText
                }
                progressBar.UpdateProgress(displayText, progress.BytesRead, length, progress.BytesPerSecond)
        })
}

func (c DigitizeCommand) formatUri(baseUri url.URL, org string, tenant string, projectId string) string {
        path := baseUri.Path
        if baseUri.Path == "" {
                path = "/{organization}/{tenant}/du_/api/framework/projects/{projectId}"
        }
        path = strings.ReplaceAll(path, "{organization}", org)
        path = strings.ReplaceAll(path, "{tenant}", tenant)
        path = strings.ReplaceAll(path, "{projectId}", projectId)
        path = strings.TrimSuffix(path, "/")
        return fmt.Sprintf("%s://%s%s", baseUri.Scheme, baseUri.Host, path)
}

func (c DigitizeCommand) createDigitizeStatusRequest(documentId string, ctx plugin.ExecutionContext) *network.HttpRequest {
        projectId := c.getProjectId(ctx.Parameters)
        uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + fmt.Sprintf("/digitization/result/%s?api-version=1", documentId)
        header := http.Header{}
        for key, value := range ctx.Auth.Header {
                header.Set(key, value)
        }
        return network.NewHttpRequest(http.MethodGet, uri, header, &bytes.Buffer{})
}

func (c DigitizeCommand) calculateMultipartSize(stream stream.Stream) int64 {
        size, _ := stream.Size()
        return size
}

func (c DigitizeCommand) writeMultipartForm(writer *multipart.Writer, stream stream.Stream, contentType string) error {
        filePart := textproto.MIMEHeader{}
        filePart.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, stream.Name()))
        filePart.Set("Content-Type", contentType)
        w, err := writer.CreatePart(filePart)
        if err != nil {
                return fmt.Errorf("Error creating form field 'file': %w", err)
        }
        data, err := stream.Data()
        if err != nil {
                return err
        }
        defer data.Close()
        _, err = io.Copy(w, data)
        if err != nil {
                return fmt.Errorf("Error writing form field 'file': %w", err)
        }
        return nil
}

func (c DigitizeCommand) writeMultipartBody(bodyWriter *io.PipeWriter, stream stream.Stream, contentType string, cancel context.CancelCauseFunc) (string, int64) {
        contentLength := c.calculateMultipartSize(stream)
        formWriter := multipart.NewWriter(bodyWriter)
        go func() {
                defer bodyWriter.Close()
                defer formWriter.Close()
                err := c.writeMultipartForm(formWriter, stream, contentType)
                if err != nil {
                        cancel(err)
                        return
                }
        }()
        return formWriter.FormDataContentType(), contentLength
}

func (c DigitizeCommand) getProjectId(parameters []plugin.ExecutionParameter) string {
        projectId := c.getParameter("project-id", parameters)
        if projectId == "" {
                projectId = "00000000-0000-0000-0000-000000000000"
        }
        return projectId
}

func (c DigitizeCommand) getParameter(name string, parameters []plugin.ExecutionParameter) string {
        result := ""
        for _, p := range parameters {
                if p.Name == name {
                        if data, ok := p.Value.(string); ok {
                                result = data
                                break
                        }
                }
        }
        return result
}

func (c DigitizeCommand) getFileParameter(parameters []plugin.ExecutionParameter) stream.Stream {
        var result stream.Stream
        for _, p := range parameters {
                if p.Name == "file" {
                        if stream, ok := p.Value.(stream.Stream); ok {
                                result = stream
                                break
                        }
                }
        }
        return result
}

func (c DigitizeCommand) httpClientSettings(ctx plugin.ExecutionContext) network.HttpClientSettings {
        return *network.NewHttpClientSettings(
                ctx.Debug,
                ctx.Settings.OperationId,
                ctx.Settings.Timeout,
                ctx.Settings.MaxAttempts,
                ctx.Settings.Insecure)
}

func NewDigitizeCommand() *DigitizeCommand {
        return &DigitizeCommand{}
}

1	package digitzer
2
3	import (
4	"bytes"
5	"context"
6	"encoding/json"
7	"errors"
8	"fmt"
9	"io"
10	"mime/multipart"
11	"net/http"
12	"net/textproto"
13	"net/url"
14	"strings"
15	"time"
16
17	"github.com/UiPath/uipathcli/log"
18	"github.com/UiPath/uipathcli/output"
19	"github.com/UiPath/uipathcli/plugin"
20	"github.com/UiPath/uipathcli/utils/network"
21	"github.com/UiPath/uipathcli/utils/stream"
22	"github.com/UiPath/uipathcli/utils/visualization"
23	)
24
25	// The DigitizeCommand is a convenient wrapper over the async digitizer API
26	// to make it seem like it is a single sync call.
27	type DigitizeCommand struct{}
28
29	func (c DigitizeCommand) Command() plugin.Command {	1✔
30	return *plugin.NewCommand("du").	1✔
31	WithCategory("digitization", "Document Digitization", "Digitizes a document, extracting its Document Object Model (DOM) and text.").	1✔
32	WithOperation("digitize", "Digitize file", "Digitize the given file").	1✔
33	WithParameter("project-id", plugin.ParameterTypeString, "The project id", false).	1✔
34	WithParameter("file", plugin.ParameterTypeBinary, "The file to digitize", true).	1✔
35	WithParameter("content-type", plugin.ParameterTypeString, "The content type", false)	1✔
36	}	1✔
37
38	func (c DigitizeCommand) Execute(ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) error {	1✔
39	if ctx.Organization == "" {	2✔
40	return errors.New("Organization is not set")	1✔
41	}	1✔
42	if ctx.Tenant == "" {	2✔
43	return errors.New("Tenant is not set")	1✔
44	}	1✔
45	documentId, err := c.startDigitization(ctx, logger)	1✔
46	if err != nil {	2✔
47	return err	1✔
48	}	1✔
49
50	for i := 1; i <= 60; i++ {	2✔
51	finished, err := c.waitForDigitization(documentId, ctx, writer, logger)	1✔
52	if err != nil {	2✔
53	return err	1✔
54	}	1✔
55	if finished {	2✔
56	return nil	1✔
57	}	1✔
58	time.Sleep(1 * time.Second)	×
59	}
60	return fmt.Errorf("Digitization with documentId '%s' did not finish in time", documentId)	×
61	}
62
63	func (c DigitizeCommand) startDigitization(ctx plugin.ExecutionContext, logger log.Logger) (string, error) {	1✔
64	uploadBar := visualization.NewProgressBar(logger)	1✔
65	defer uploadBar.Remove()	1✔
66	context, cancel := context.WithCancelCause(context.Background())	1✔
67	request := c.createDigitizeRequest(ctx, uploadBar, cancel)	1✔
68	client := network.NewHttpClient(logger, c.httpClientSettings(ctx))	1✔
69	response, err := client.SendWithContext(request, context)	1✔
70	if err != nil {	2✔
71	return "", err	1✔
72	}	1✔
73	defer response.Body.Close()	1✔
74	body, err := io.ReadAll(response.Body)	1✔
75	if err != nil {	1✔
76	return "", fmt.Errorf("Error reading response: %w", err)	×
77	}	×
78	if response.StatusCode != http.StatusAccepted {	2✔
79	return "", fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))	1✔
80	}	1✔
81	var result digitizeResponse	1✔
82	err = json.Unmarshal(body, &result)	1✔
83	if err != nil {	1✔
84	return "", fmt.Errorf("Error parsing json response: %w", err)	×
85	}	×
86	return result.DocumentId, nil	1✔
87	}
88
89	func (c DigitizeCommand) waitForDigitization(documentId string, ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) (bool, error) {	1✔
90	request := c.createDigitizeStatusRequest(documentId, ctx)	1✔
91	client := network.NewHttpClient(logger, c.httpClientSettings(ctx))	1✔
92	response, err := client.Send(request)	1✔
93	if err != nil {	1✔
94	return true, err	×
95	}	×
96	defer response.Body.Close()	1✔
97	body, err := io.ReadAll(response.Body)	1✔
98	if err != nil {	1✔
99	return true, fmt.Errorf("Error reading response: %w", err)	×
100	}	×
101	if response.StatusCode != http.StatusOK {	2✔
102	return true, fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))	1✔
103	}	1✔
104	var result digitizeResultResponse	1✔
105	err = json.Unmarshal(body, &result)	1✔
106	if err != nil {	1✔
107	return true, fmt.Errorf("Error parsing json response: %w", err)	×
108	}	×
109	if result.Status == "NotStarted" \|\| result.Status == "Running" {	1✔
110	return false, nil	×
111	}	×
112	err = writer.WriteResponse(*output.NewResponseInfo(response.StatusCode, response.Status, response.Proto, response.Header, bytes.NewReader(body)))	1✔
113	return true, err	1✔
114	}
115
116	func (c DigitizeCommand) createDigitizeRequest(ctx plugin.ExecutionContext, uploadBar visualization.ProgressBar, cancel context.CancelCauseFunc) network.HttpRequest {	1✔
117	projectId := c.getProjectId(ctx.Parameters)	1✔
118		1✔
119	file := ctx.Input	1✔
120	if file == nil {	2✔
121	file = c.getFileParameter(ctx.Parameters)	1✔
122	}	1✔
123	contentType := c.getParameter("content-type", ctx.Parameters)	1✔
124	if contentType == "" {	2✔
125	contentType = "application/octet-stream"	1✔
126	}	1✔
127
128	bodyReader, bodyWriter := io.Pipe()	1✔
129	contentType, contentLength := c.writeMultipartBody(bodyWriter, file, contentType, cancel)	1✔
130	uploadReader := c.progressReader("uploading...", "completing ", bodyReader, contentLength, uploadBar)	1✔
131		1✔
132	uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + "/digitization/start?api-version=1"	1✔
133	header := http.Header{	1✔
134	"Content-Type": {contentType},	1✔
135	}	1✔
136	for key, value := range ctx.Auth.Header {	1✔
NEW 137	header.Set(key, value)	×
UNCOV 138	}	×
139	return network.NewHttpRequest(http.MethodPost, uri, header, uploadReader)	1✔
140	}
141
142	func (c DigitizeCommand) progressReader(text string, completedText string, reader io.Reader, length int64, progressBar *visualization.ProgressBar) io.Reader {	1✔
143	if length < 1010241024 {	2✔
144	return reader	1✔
145	}	1✔
146	return visualization.NewProgressReader(reader, func(progress visualization.Progress) {	2✔
147	displayText := text	1✔
148	if progress.Completed {	2✔
149	displayText = completedText	1✔
150	}	1✔
151	progressBar.UpdateProgress(displayText, progress.BytesRead, length, progress.BytesPerSecond)	1✔
152	})
153	}
154
155	func (c DigitizeCommand) formatUri(baseUri url.URL, org string, tenant string, projectId string) string {	1✔
156	path := baseUri.Path	1✔
157	if baseUri.Path == "" {	2✔
158	path = "/{organization}/{tenant}/du_/api/framework/projects/{projectId}"	1✔
159	}	1✔
160	path = strings.ReplaceAll(path, "{organization}", org)	1✔
161	path = strings.ReplaceAll(path, "{tenant}", tenant)	1✔
162	path = strings.ReplaceAll(path, "{projectId}", projectId)	1✔
163	path = strings.TrimSuffix(path, "/")	1✔
164	return fmt.Sprintf("%s://%s%s", baseUri.Scheme, baseUri.Host, path)	1✔
165	}
166
167	func (c DigitizeCommand) createDigitizeStatusRequest(documentId string, ctx plugin.ExecutionContext) *network.HttpRequest {	1✔
168	projectId := c.getProjectId(ctx.Parameters)	1✔
169	uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + fmt.Sprintf("/digitization/result/%s?api-version=1", documentId)	1✔
170	header := http.Header{}	1✔
171	for key, value := range ctx.Auth.Header {	1✔
NEW 172	header.Set(key, value)	×
NEW 173	}	×
174	return network.NewHttpRequest(http.MethodGet, uri, header, &bytes.Buffer{})	1✔
175	}
176
177	func (c DigitizeCommand) calculateMultipartSize(stream stream.Stream) int64 {	1✔
178	size, _ := stream.Size()	1✔
179	return size	1✔
180	}	1✔
181
182	func (c DigitizeCommand) writeMultipartForm(writer *multipart.Writer, stream stream.Stream, contentType string) error {	1✔
183	filePart := textproto.MIMEHeader{}	1✔
184	filePart.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, stream.Name()))	1✔
185	filePart.Set("Content-Type", contentType)	1✔
186	w, err := writer.CreatePart(filePart)	1✔
187	if err != nil {	1✔
188	return fmt.Errorf("Error creating form field 'file': %w", err)	×
189	}	×
190	data, err := stream.Data()	1✔
191	if err != nil {	2✔
192	return err	1✔
193	}	1✔
194	defer data.Close()	1✔
195	_, err = io.Copy(w, data)	1✔
196	if err != nil {	1✔
197	return fmt.Errorf("Error writing form field 'file': %w", err)	×
198	}	×
199	return nil	1✔
200	}
201
202	func (c DigitizeCommand) writeMultipartBody(bodyWriter *io.PipeWriter, stream stream.Stream, contentType string, cancel context.CancelCauseFunc) (string, int64) {	1✔
203	contentLength := c.calculateMultipartSize(stream)	1✔
204	formWriter := multipart.NewWriter(bodyWriter)	1✔
205	go func() {	2✔
206	defer bodyWriter.Close()	1✔
207	defer formWriter.Close()	1✔
208	err := c.writeMultipartForm(formWriter, stream, contentType)	1✔
209	if err != nil {	2✔
210	cancel(err)	1✔
211	return	1✔
212	}	1✔
213	}()
214	return formWriter.FormDataContentType(), contentLength	1✔
215	}
216
217	func (c DigitizeCommand) getProjectId(parameters []plugin.ExecutionParameter) string {	1✔
218	projectId := c.getParameter("project-id", parameters)	1✔
219	if projectId == "" {	2✔
220	projectId = "00000000-0000-0000-0000-000000000000"	1✔
221	}	1✔
222	return projectId	1✔
223	}
224
225	func (c DigitizeCommand) getParameter(name string, parameters []plugin.ExecutionParameter) string {	1✔
226	result := ""	1✔
227	for _, p := range parameters {	2✔
228	if p.Name == name {	2✔
229	if data, ok := p.Value.(string); ok {	2✔
230	result = data	1✔
231	break	1✔
232	}
233	}
234	}
235	return result	1✔
236	}
237
238	func (c DigitizeCommand) getFileParameter(parameters []plugin.ExecutionParameter) stream.Stream {	1✔
239	var result stream.Stream	1✔
240	for _, p := range parameters {	2✔
241	if p.Name == "file" {	2✔
242	if stream, ok := p.Value.(stream.Stream); ok {	2✔
243	result = stream	1✔
244	break	1✔
245	}
246	}
247	}
248	return result	1✔
249	}
250
251	func (c DigitizeCommand) httpClientSettings(ctx plugin.ExecutionContext) network.HttpClientSettings {	1✔
252	return *network.NewHttpClientSettings(	1✔
253	ctx.Debug,	1✔
254	ctx.Settings.OperationId,	1✔
255	ctx.Settings.Timeout,	1✔
256	ctx.Settings.MaxAttempts,	1✔
257	ctx.Settings.Insecure)	1✔
258	}	1✔
259
260	func NewDigitizeCommand() *DigitizeCommand {	1✔
261	return &DigitizeCommand{}	1✔
262	}	1✔

UiPath / uipathcli / 13790794198

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous