13765865558

Committed 10 Mar 2025 01:27PM UTC coverage: 90.218% (+0.1%) from 90.121%

Build # 13765865558

Build Type

push

github

Committed by

thschmitt

Commit Message

Add network package with common http client

Moved common code for handling HTTP requests in a new shared
network package to avoid duplication and ensure that each plugin
handles network requests the same way.

The common HTTP client code takes care of a couple of cross-cutting
concerns needed by all plugins and the main HTTP executor:

- Logs request and response when debug flag is set. Added a new
  resettableReader which preserves the request and response bodies while
  they are being read and forwards them to the debug logger.

- Added retries for all HTTP requests. This also leverages the
  resettableReader to ensure the request body can be replayed.

- The `CommandBuilder` generates now an operation id which is set on every
  request the uipathcli performs. The same operation id is kept for the
  duration of the whole command execution which makes it easier to
  correlate multiple requests performed by a single command.

- The `HttpClient` also sets transport-related settings like certificate
  validation and response header timeout.

- Using the built-in context instead of a custom requestError channel.

Run Details

369 of 384 new or added lines in 22 files covered. (96.09%)

7 existing lines in 4 files now uncovered.

5340 of 5919 relevant lines covered (90.22%)

1.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.65

/plugin/digitizer/digitize_command.go

package digitzer

import (
        "bytes"
        "context"
        "encoding/json"
        "errors"
        "fmt"
        "io"
        "mime/multipart"
        "net/http"
        "net/textproto"
        "net/url"
        "strings"
        "time"

        "github.com/UiPath/uipathcli/log"
        "github.com/UiPath/uipathcli/output"
        "github.com/UiPath/uipathcli/plugin"
        "github.com/UiPath/uipathcli/utils/network"
        "github.com/UiPath/uipathcli/utils/stream"
        "github.com/UiPath/uipathcli/utils/visualization"
)

// The DigitizeCommand is a convenient wrapper over the async digitizer API
// to make it seem like it is a single sync call.
type DigitizeCommand struct{}

func (c DigitizeCommand) Command() plugin.Command {
        return *plugin.NewCommand("du").
                WithCategory("digitization", "Document Digitization", "Digitizes a document, extracting its Document Object Model (DOM) and text.").
                WithOperation("digitize", "Digitize file", "Digitize the given file").
                WithParameter("project-id", plugin.ParameterTypeString, "The project id", false).
                WithParameter("file", plugin.ParameterTypeBinary, "The file to digitize", true).
                WithParameter("content-type", plugin.ParameterTypeString, "The content type", false)
}

func (c DigitizeCommand) Execute(ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) error {
        if ctx.Organization == "" {
                return errors.New("Organization is not set")
        }
        if ctx.Tenant == "" {
                return errors.New("Tenant is not set")
        }
        documentId, err := c.startDigitization(ctx, logger)
        if err != nil {
                return err
        }

        for i := 1; i <= 60; i++ {
                finished, err := c.waitForDigitization(documentId, ctx, writer, logger)
                if err != nil {
                        return err
                }
                if finished {
                        return nil
                }
                time.Sleep(1 * time.Second)
        }
        return fmt.Errorf("Digitization with documentId '%s' did not finish in time", documentId)
}

func (c DigitizeCommand) startDigitization(ctx plugin.ExecutionContext, logger log.Logger) (string, error) {
        uploadBar := visualization.NewProgressBar(logger)
        defer uploadBar.Remove()
        context, cancel := context.WithCancelCause(context.Background())
        request, err := c.createDigitizeRequest(ctx, uploadBar, cancel)
        if err != nil {
                return "", err
        }
        client := network.NewHttpClient(logger, c.httpClientSettings(ctx))
        response, err := client.SendWithContext(request, context)
        if err != nil {
                return "", err
        }
        defer response.Body.Close()
        body, err := io.ReadAll(response.Body)
        if err != nil {
                return "", fmt.Errorf("Error reading response: %w", err)
        }
        if response.StatusCode != http.StatusAccepted {
                return "", fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))
        }
        var result digitizeResponse
        err = json.Unmarshal(body, &result)
        if err != nil {
                return "", fmt.Errorf("Error parsing json response: %w", err)
        }
        return result.DocumentId, nil
}

func (c DigitizeCommand) waitForDigitization(documentId string, ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) (bool, error) {
        request, err := c.createDigitizeStatusRequest(documentId, ctx)
        if err != nil {
                return true, err
        }
        client := network.NewHttpClient(logger, c.httpClientSettings(ctx))
        response, err := client.Send(request)
        if err != nil {
                return true, err
        }
        defer response.Body.Close()
        body, err := io.ReadAll(response.Body)
        if err != nil {
                return true, fmt.Errorf("Error reading response: %w", err)
        }
        if response.StatusCode != http.StatusOK {
                return true, fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))
        }
        var result digitizeResultResponse
        err = json.Unmarshal(body, &result)
        if err != nil {
                return true, fmt.Errorf("Error parsing json response: %w", err)
        }
        if result.Status == "NotStarted" || result.Status == "Running" {
                return false, nil
        }
        err = writer.WriteResponse(*output.NewResponseInfo(response.StatusCode, response.Status, response.Proto, response.Header, bytes.NewReader(body)))
        return true, err
}

func (c DigitizeCommand) createDigitizeRequest(ctx plugin.ExecutionContext, uploadBar *visualization.ProgressBar, cancel context.CancelCauseFunc) (*http.Request, error) {
        projectId := c.getProjectId(ctx.Parameters)

        var err error
        file := ctx.Input
        if file == nil {
                file = c.getFileParameter(ctx.Parameters)
        }
        contentType := c.getParameter("content-type", ctx.Parameters)
        if contentType == "" {
                contentType = "application/octet-stream"
        }

        bodyReader, bodyWriter := io.Pipe()
        contentType, contentLength := c.writeMultipartBody(bodyWriter, file, contentType, cancel)
        uploadReader := c.progressReader("uploading...", "completing  ", bodyReader, contentLength, uploadBar)

        uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + "/digitization/start?api-version=1"
        request, err := http.NewRequest("POST", uri, uploadReader)
        if err != nil {
                return nil, err
        }
        request.Header.Add("Content-Type", contentType)
        for key, value := range ctx.Auth.Header {
                request.Header.Add(key, value)
        }
        return request, nil
}

func (c DigitizeCommand) progressReader(text string, completedText string, reader io.Reader, length int64, progressBar *visualization.ProgressBar) io.Reader {
        if length < 10*1024*1024 {
                return reader
        }
        progressReader := visualization.NewProgressReader(reader, func(progress visualization.Progress) {
                displayText := text
                if progress.Completed {
                        displayText = completedText
                }
                progressBar.UpdateProgress(displayText, progress.BytesRead, length, progress.BytesPerSecond)
        })
        return progressReader
}

func (c DigitizeCommand) formatUri(baseUri url.URL, org string, tenant string, projectId string) string {
        path := baseUri.Path
        if baseUri.Path == "" {
                path = "/{organization}/{tenant}/du_/api/framework/projects/{projectId}"
        }
        path = strings.ReplaceAll(path, "{organization}", org)
        path = strings.ReplaceAll(path, "{tenant}", tenant)
        path = strings.ReplaceAll(path, "{projectId}", projectId)
        path = strings.TrimSuffix(path, "/")
        return fmt.Sprintf("%s://%s%s", baseUri.Scheme, baseUri.Host, path)
}

func (c DigitizeCommand) createDigitizeStatusRequest(documentId string, ctx plugin.ExecutionContext) (*http.Request, error) {
        projectId := c.getProjectId(ctx.Parameters)
        uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + fmt.Sprintf("/digitization/result/%s?api-version=1", documentId)
        request, err := http.NewRequest("GET", uri, &bytes.Buffer{})
        if err != nil {
                return nil, err
        }
        for key, value := range ctx.Auth.Header {
                request.Header.Add(key, value)
        }
        return request, nil
}

func (c DigitizeCommand) calculateMultipartSize(stream stream.Stream) int64 {
        size, _ := stream.Size()
        return size
}

func (c DigitizeCommand) writeMultipartForm(writer *multipart.Writer, stream stream.Stream, contentType string) error {
        filePart := textproto.MIMEHeader{}
        filePart.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, stream.Name()))
        filePart.Set("Content-Type", contentType)
        w, err := writer.CreatePart(filePart)
        if err != nil {
                return fmt.Errorf("Error creating form field 'file': %w", err)
        }
        data, err := stream.Data()
        if err != nil {
                return err
        }
        defer data.Close()
        _, err = io.Copy(w, data)
        if err != nil {
                return fmt.Errorf("Error writing form field 'file': %w", err)
        }
        return nil
}

func (c DigitizeCommand) writeMultipartBody(bodyWriter *io.PipeWriter, stream stream.Stream, contentType string, cancel context.CancelCauseFunc) (string, int64) {
        contentLength := c.calculateMultipartSize(stream)
        formWriter := multipart.NewWriter(bodyWriter)
        go func() {
                defer bodyWriter.Close()
                defer formWriter.Close()
                err := c.writeMultipartForm(formWriter, stream, contentType)
                if err != nil {
                        cancel(err)
                        return
                }
        }()
        return formWriter.FormDataContentType(), contentLength
}

func (c DigitizeCommand) getProjectId(parameters []plugin.ExecutionParameter) string {
        projectId := c.getParameter("project-id", parameters)
        if projectId == "" {
                projectId = "00000000-0000-0000-0000-000000000000"
        }
        return projectId
}

func (c DigitizeCommand) getParameter(name string, parameters []plugin.ExecutionParameter) string {
        result := ""
        for _, p := range parameters {
                if p.Name == name {
                        if data, ok := p.Value.(string); ok {
                                result = data
                                break
                        }
                }
        }
        return result
}

func (c DigitizeCommand) getFileParameter(parameters []plugin.ExecutionParameter) stream.Stream {
        var result stream.Stream
        for _, p := range parameters {
                if p.Name == "file" {
                        if stream, ok := p.Value.(stream.Stream); ok {
                                result = stream
                                break
                        }
                }
        }
        return result
}

func (c DigitizeCommand) httpClientSettings(ctx plugin.ExecutionContext) network.HttpClientSettings {
        return *network.NewHttpClientSettings(
                ctx.Debug,
                ctx.Settings.OperationId,
                ctx.Settings.Timeout,
                ctx.Settings.MaxAttempts,
                ctx.Settings.Insecure)
}

func NewDigitizeCommand() *DigitizeCommand {
        return &DigitizeCommand{}
}

1	package digitzer
2
3	import (
4	"bytes"
5	"context"
6	"encoding/json"
7	"errors"
8	"fmt"
9	"io"
10	"mime/multipart"
11	"net/http"
12	"net/textproto"
13	"net/url"
14	"strings"
15	"time"
16
17	"github.com/UiPath/uipathcli/log"
18	"github.com/UiPath/uipathcli/output"
19	"github.com/UiPath/uipathcli/plugin"
20	"github.com/UiPath/uipathcli/utils/network"
21	"github.com/UiPath/uipathcli/utils/stream"
22	"github.com/UiPath/uipathcli/utils/visualization"
23	)
24
25	// The DigitizeCommand is a convenient wrapper over the async digitizer API
26	// to make it seem like it is a single sync call.
27	type DigitizeCommand struct{}
28
29	func (c DigitizeCommand) Command() plugin.Command {	1✔
30	return *plugin.NewCommand("du").	1✔
31	WithCategory("digitization", "Document Digitization", "Digitizes a document, extracting its Document Object Model (DOM) and text.").	1✔
32	WithOperation("digitize", "Digitize file", "Digitize the given file").	1✔
33	WithParameter("project-id", plugin.ParameterTypeString, "The project id", false).	1✔
34	WithParameter("file", plugin.ParameterTypeBinary, "The file to digitize", true).	1✔
35	WithParameter("content-type", plugin.ParameterTypeString, "The content type", false)	1✔
36	}	1✔
37
38	func (c DigitizeCommand) Execute(ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) error {	1✔
39	if ctx.Organization == "" {	2✔
40	return errors.New("Organization is not set")	1✔
41	}	1✔
42	if ctx.Tenant == "" {	2✔
43	return errors.New("Tenant is not set")	1✔
44	}	1✔
45	documentId, err := c.startDigitization(ctx, logger)	1✔
46	if err != nil {	2✔
47	return err	1✔
48	}	1✔
49
50	for i := 1; i <= 60; i++ {	2✔
51	finished, err := c.waitForDigitization(documentId, ctx, writer, logger)	1✔
52	if err != nil {	2✔
53	return err	1✔
54	}	1✔
55	if finished {	2✔
56	return nil	1✔
57	}	1✔
58	time.Sleep(1 * time.Second)	×
59	}
60	return fmt.Errorf("Digitization with documentId '%s' did not finish in time", documentId)	×
61	}
62
63	func (c DigitizeCommand) startDigitization(ctx plugin.ExecutionContext, logger log.Logger) (string, error) {	1✔
64	uploadBar := visualization.NewProgressBar(logger)	1✔
65	defer uploadBar.Remove()	1✔
66	context, cancel := context.WithCancelCause(context.Background())	1✔
67	request, err := c.createDigitizeRequest(ctx, uploadBar, cancel)	1✔
68	if err != nil {	1✔
69	return "", err	×
70	}	×
71	client := network.NewHttpClient(logger, c.httpClientSettings(ctx))	1✔
72	response, err := client.SendWithContext(request, context)	1✔
73	if err != nil {	2✔
74	return "", err	1✔
75	}	1✔
76	defer response.Body.Close()	1✔
77	body, err := io.ReadAll(response.Body)	1✔
78	if err != nil {	1✔
79	return "", fmt.Errorf("Error reading response: %w", err)	×
80	}	×
81	if response.StatusCode != http.StatusAccepted {	2✔
82	return "", fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))	1✔
83	}	1✔
84	var result digitizeResponse	1✔
85	err = json.Unmarshal(body, &result)	1✔
86	if err != nil {	1✔
87	return "", fmt.Errorf("Error parsing json response: %w", err)	×
88	}	×
89	return result.DocumentId, nil	1✔
90	}
91
92	func (c DigitizeCommand) waitForDigitization(documentId string, ctx plugin.ExecutionContext, writer output.OutputWriter, logger log.Logger) (bool, error) {	1✔
93	request, err := c.createDigitizeStatusRequest(documentId, ctx)	1✔
94	if err != nil {	1✔
95	return true, err	×
96	}	×
97	client := network.NewHttpClient(logger, c.httpClientSettings(ctx))	1✔
98	response, err := client.Send(request)	1✔
99	if err != nil {	1✔
NEW 100	return true, err	×
101	}	×
102	defer response.Body.Close()	1✔
103	body, err := io.ReadAll(response.Body)	1✔
104	if err != nil {	1✔
105	return true, fmt.Errorf("Error reading response: %w", err)	×
106	}	×
107	if response.StatusCode != http.StatusOK {	2✔
108	return true, fmt.Errorf("Digitizer returned status code '%v' and body '%v'", response.StatusCode, string(body))	1✔
109	}	1✔
110	var result digitizeResultResponse	1✔
111	err = json.Unmarshal(body, &result)	1✔
112	if err != nil {	1✔
113	return true, fmt.Errorf("Error parsing json response: %w", err)	×
114	}	×
115	if result.Status == "NotStarted" \|\| result.Status == "Running" {	1✔
116	return false, nil	×
117	}	×
118	err = writer.WriteResponse(*output.NewResponseInfo(response.StatusCode, response.Status, response.Proto, response.Header, bytes.NewReader(body)))	1✔
119	return true, err	1✔
120	}
121
122	func (c DigitizeCommand) createDigitizeRequest(ctx plugin.ExecutionContext, uploadBar visualization.ProgressBar, cancel context.CancelCauseFunc) (http.Request, error) {	1✔
123	projectId := c.getProjectId(ctx.Parameters)	1✔
124		1✔
125	var err error	1✔
126	file := ctx.Input	1✔
127	if file == nil {	2✔
128	file = c.getFileParameter(ctx.Parameters)	1✔
129	}	1✔
130	contentType := c.getParameter("content-type", ctx.Parameters)	1✔
131	if contentType == "" {	2✔
132	contentType = "application/octet-stream"	1✔
133	}	1✔
134
135	bodyReader, bodyWriter := io.Pipe()	1✔
136	contentType, contentLength := c.writeMultipartBody(bodyWriter, file, contentType, cancel)	1✔
137	uploadReader := c.progressReader("uploading...", "completing ", bodyReader, contentLength, uploadBar)	1✔
138		1✔
139	uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + "/digitization/start?api-version=1"	1✔
140	request, err := http.NewRequest("POST", uri, uploadReader)	1✔
141	if err != nil {	1✔
142	return nil, err	×
143	}	×
144	request.Header.Add("Content-Type", contentType)	1✔
145	for key, value := range ctx.Auth.Header {	1✔
146	request.Header.Add(key, value)	×
147	}	×
148	return request, nil	1✔
149	}
150
151	func (c DigitizeCommand) progressReader(text string, completedText string, reader io.Reader, length int64, progressBar *visualization.ProgressBar) io.Reader {	1✔
152	if length < 1010241024 {	2✔
153	return reader	1✔
154	}	1✔
155	progressReader := visualization.NewProgressReader(reader, func(progress visualization.Progress) {	2✔
156	displayText := text	1✔
157	if progress.Completed {	2✔
158	displayText = completedText	1✔
159	}	1✔
160	progressBar.UpdateProgress(displayText, progress.BytesRead, length, progress.BytesPerSecond)	1✔
161	})
162	return progressReader	1✔
163	}
164
165	func (c DigitizeCommand) formatUri(baseUri url.URL, org string, tenant string, projectId string) string {	1✔
166	path := baseUri.Path	1✔
167	if baseUri.Path == "" {	2✔
168	path = "/{organization}/{tenant}/du_/api/framework/projects/{projectId}"	1✔
169	}	1✔
170	path = strings.ReplaceAll(path, "{organization}", org)	1✔
171	path = strings.ReplaceAll(path, "{tenant}", tenant)	1✔
172	path = strings.ReplaceAll(path, "{projectId}", projectId)	1✔
173	path = strings.TrimSuffix(path, "/")	1✔
174	return fmt.Sprintf("%s://%s%s", baseUri.Scheme, baseUri.Host, path)	1✔
175	}
176
177	func (c DigitizeCommand) createDigitizeStatusRequest(documentId string, ctx plugin.ExecutionContext) (*http.Request, error) {	1✔
178	projectId := c.getProjectId(ctx.Parameters)	1✔
179	uri := c.formatUri(ctx.BaseUri, ctx.Organization, ctx.Tenant, projectId) + fmt.Sprintf("/digitization/result/%s?api-version=1", documentId)	1✔
180	request, err := http.NewRequest("GET", uri, &bytes.Buffer{})	1✔
181	if err != nil {	1✔
182	return nil, err	×
183	}	×
184	for key, value := range ctx.Auth.Header {	1✔
185	request.Header.Add(key, value)	×
186	}	×
187	return request, nil	1✔
188	}
189
190	func (c DigitizeCommand) calculateMultipartSize(stream stream.Stream) int64 {	1✔
191	size, _ := stream.Size()	1✔
192	return size	1✔
193	}	1✔
194
195	func (c DigitizeCommand) writeMultipartForm(writer *multipart.Writer, stream stream.Stream, contentType string) error {	1✔
196	filePart := textproto.MIMEHeader{}	1✔
197	filePart.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, stream.Name()))	1✔
198	filePart.Set("Content-Type", contentType)	1✔
199	w, err := writer.CreatePart(filePart)	1✔
200	if err != nil {	1✔
201	return fmt.Errorf("Error creating form field 'file': %w", err)	×
202	}	×
203	data, err := stream.Data()	1✔
204	if err != nil {	2✔
205	return err	1✔
206	}	1✔
207	defer data.Close()	1✔
208	_, err = io.Copy(w, data)	1✔
209	if err != nil {	1✔
210	return fmt.Errorf("Error writing form field 'file': %w", err)	×
211	}	×
212	return nil	1✔
213	}
214
215	func (c DigitizeCommand) writeMultipartBody(bodyWriter *io.PipeWriter, stream stream.Stream, contentType string, cancel context.CancelCauseFunc) (string, int64) {	1✔
216	contentLength := c.calculateMultipartSize(stream)	1✔
217	formWriter := multipart.NewWriter(bodyWriter)	1✔
218	go func() {	2✔
219	defer bodyWriter.Close()	1✔
220	defer formWriter.Close()	1✔
221	err := c.writeMultipartForm(formWriter, stream, contentType)	1✔
222	if err != nil {	2✔
223	cancel(err)	1✔
224	return	1✔
225	}	1✔
226	}()
227	return formWriter.FormDataContentType(), contentLength	1✔
228	}
229
230	func (c DigitizeCommand) getProjectId(parameters []plugin.ExecutionParameter) string {	1✔
231	projectId := c.getParameter("project-id", parameters)	1✔
232	if projectId == "" {	2✔
233	projectId = "00000000-0000-0000-0000-000000000000"	1✔
234	}	1✔
235	return projectId	1✔
236	}
237
238	func (c DigitizeCommand) getParameter(name string, parameters []plugin.ExecutionParameter) string {	1✔
239	result := ""	1✔
240	for _, p := range parameters {	2✔
241	if p.Name == name {	2✔
242	if data, ok := p.Value.(string); ok {	2✔
243	result = data	1✔
244	break	1✔
245	}
246	}
247	}
248	return result	1✔
249	}
250
251	func (c DigitizeCommand) getFileParameter(parameters []plugin.ExecutionParameter) stream.Stream {	1✔
252	var result stream.Stream	1✔
253	for _, p := range parameters {	2✔
254	if p.Name == "file" {	2✔
255	if stream, ok := p.Value.(stream.Stream); ok {	2✔
256	result = stream	1✔
257	break	1✔
258	}
259	}
260	}
261	return result	1✔
262	}
263
264	func (c DigitizeCommand) httpClientSettings(ctx plugin.ExecutionContext) network.HttpClientSettings {	1✔
265	return *network.NewHttpClientSettings(	1✔
266	ctx.Debug,	1✔
267	ctx.Settings.OperationId,	1✔
268	ctx.Settings.Timeout,	1✔
269	ctx.Settings.MaxAttempts,	1✔
270	ctx.Settings.Insecure)	1✔
271	}	1✔
272
273	func NewDigitizeCommand() *DigitizeCommand {	1✔
274	return &DigitizeCommand{}	1✔
275	}	1✔

UiPath / uipathcli / 13765865558

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous