15042063902

Committed 15 May 2025 09:56AM UTC coverage: 60.974% (-0.03%) from 61.006%

Build # 15042063902

Build Type

Pull #857

github

Committed by

web-flow

Commit Message

Merge ec4aaf1ce into 5c4b146a4

Pull Request Pull Request #857: fix(jonnetsecure): prevent massive memory usage from misbehaving jsonnet script

Run Details

53 of 81 new or added lines in 5 files covered. (65.43%)

4 existing lines in 2 files now uncovered.

7212 of 11828 relevant lines covered (60.97%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.38

/jsonnetsecure/jsonnet_processvm.go

// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0

package jsonnetsecure

import (
        "bytes"
        "context"
        "encoding/json"
        "fmt"
        "io"
        "os/exec"
        "syscall"
        "time"

        "github.com/cenkalti/backoff/v4"
        "github.com/pkg/errors"
        "go.opentelemetry.io/otel/attribute"
        "go.opentelemetry.io/otel/trace"

        "github.com/ory/x/otelx"
)

const (
        KiB                = 1024
        jsonnetOutputLimit = 512 * KiB
        jsonnetErrLimit    = 1 * KiB
)

func NewProcessVM(opts *vmOptions) VM {
        return &ProcessVM{
                path: opts.jsonnetBinaryPath,
                args: opts.args,
                ctx:  opts.ctx,
        }
}

// Jsonnet evaluation is run in a subprocess with a timeout, and
// standard output and error are limited in size.
// The subprocess is killed when a limit (whichever comes first) is reached.
// In this case an error is returned.
func (p *ProcessVM) EvaluateAnonymousSnippet(filename string, snippet string) (_ string, err error) {
        tracer := trace.SpanFromContext(p.ctx).TracerProvider().Tracer("")
        ctx, span := tracer.Start(p.ctx, "jsonnetsecure.ProcessVM.EvaluateAnonymousSnippet", trace.WithAttributes(attribute.String("filename", filename)))
        defer otelx.End(span, &err)

        // We retry the process creation, because it sometimes times out.
        const processVMTimeout = 1 * time.Second
        return backoff.RetryWithData(func() (_ string, err error) {
                ctx, span := tracer.Start(ctx, "jsonnetsecure.ProcessVM.EvaluateAnonymousSnippet.run")
                defer otelx.End(span, &err)

                ctx, cancel := context.WithTimeout(ctx, processVMTimeout)
                defer cancel()

                var (
                        stdin bytes.Buffer
                )

                p.params.Filename = filename
                p.params.Snippet = snippet

                if err := p.params.EncodeTo(&stdin); err != nil {
                        return "", backoff.Permanent(errors.WithStack(err))
                }

                cmd := exec.CommandContext(ctx, p.path, p.args...) //nolint:gosec
                cmd.Stdin = &stdin
                cmd.Env = []string{"GOMAXPROCS=1"}
                cmd.WaitDelay = 100 * time.Millisecond

                stdoutPipe, err := cmd.StdoutPipe()
                if err != nil {
                        return "", backoff.Permanent(errors.WithStack(err))
                }
                defer stdoutPipe.Close()

                stderrPipe, err := cmd.StderrPipe()
                if err != nil {
                        return "", backoff.Permanent(errors.WithStack(err))
                }
                defer stderrPipe.Close()

                stdoutReader := io.LimitReader(stdoutPipe, jsonnetOutputLimit)
                stderrReader := io.LimitReader(stderrPipe, jsonnetErrLimit)

                if err := cmd.Start(); err != nil {
                        if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.ENOMEM) {
                                return "", errors.WithStack(fmt.Errorf("jsonnetsecure: failed to start subprocess, retrying %v", err))
                        }
                        return "", backoff.Permanent(fmt.Errorf("jsonnetsecure: failed to start subprocess: %w", err))
                }

                stdoutOutput, err := io.ReadAll(stdoutReader)
                if err != nil {
                        return "", backoff.Permanent(fmt.Errorf("jsonnetsecure: failed to read subprocess stdout: %w", err))
                }

                // Reading from stderr is best effort (there might not be anything).
                stderrOutput, _ := io.ReadAll(stderrReader)

                // If there was some stderr output or the stdout has reached the limit,
                // no point in keeping the subprocess running so we kill it.
                // This limits the negative effect of misbehaving jsonnet scripts.
                // NOTE: Depending on what the subprocess does and the OS scheduling, this might kill the subprocess, or have no effect (e.g. the child already terminated).
                if len(stderrOutput) > 0 || len(stdoutOutput) == int(jsonnetOutputLimit) {
                        cmd.Cancel()

                        return "", backoff.Permanent(fmt.Errorf("jsonnetsecure: subprocess encountered an error or reached limits: stderr=%s stdout_length=%d", string(stderrOutput), len(stdoutOutput)))
                }

                err = cmd.Wait()
                if err != nil || len(stderrOutput) > 0 {
                        return "", backoff.Permanent(fmt.Errorf("jsonnetsecure: subprocess encountered an error: %w %s", err, string(stderrOutput)))
                }

                return string(stdoutOutput), nil
        }, backoff.WithContext(backoff.NewExponentialBackOff(), ctx))
}

func (p *ProcessVM) ExtCode(key string, val string) {
        p.params.ExtCodes = append(p.params.ExtCodes, kv{key, val})
}

func (p *ProcessVM) ExtVar(key string, val string) {
        p.params.ExtVars = append(p.params.ExtVars, kv{key, val})
}

func (p *ProcessVM) TLACode(key string, val string) {
        p.params.TLACodes = append(p.params.TLACodes, kv{key, val})
}

func (p *ProcessVM) TLAVar(key string, val string) {
        p.params.TLAVars = append(p.params.TLAVars, kv{key, val})
}

func (pp *processParameters) EncodeTo(w io.Writer) error {
        return json.NewEncoder(w).Encode(pp)
}

func (pp *processParameters) DecodeFrom(r io.Reader) error {
        return json.NewDecoder(r).Decode(pp)
}

func (pp *processParameters) Decode(d []byte) error {
        return json.Unmarshal(d, pp)
}

1	// Copyright © 2023 Ory Corp
2	// SPDX-License-Identifier: Apache-2.0
3
4	package jsonnetsecure
5
6	import (
7	"bytes"
8	"context"
9	"encoding/json"
10	"fmt"
11	"io"
12	"os/exec"
13	"syscall"
14	"time"
15
16	"github.com/cenkalti/backoff/v4"
17	"github.com/pkg/errors"
18	"go.opentelemetry.io/otel/attribute"
19	"go.opentelemetry.io/otel/trace"
20
21	"github.com/ory/x/otelx"
22	)
23
24	const (
25	KiB = 1024
26	jsonnetOutputLimit = 512 * KiB
27	jsonnetErrLimit = 1 * KiB
28	)
29
30	func NewProcessVM(opts *vmOptions) VM {	1✔
31	return &ProcessVM{	1✔
32	path: opts.jsonnetBinaryPath,	1✔
33	args: opts.args,	1✔
34	ctx: opts.ctx,	1✔
35	}	1✔
36	}	1✔
37
38	// Jsonnet evaluation is run in a subprocess with a timeout, and
39	// standard output and error are limited in size.
40	// The subprocess is killed when a limit (whichever comes first) is reached.
41	// In this case an error is returned.
42	func (p *ProcessVM) EvaluateAnonymousSnippet(filename string, snippet string) (_ string, err error) {	1✔
43	tracer := trace.SpanFromContext(p.ctx).TracerProvider().Tracer("")	1✔
44	ctx, span := tracer.Start(p.ctx, "jsonnetsecure.ProcessVM.EvaluateAnonymousSnippet", trace.WithAttributes(attribute.String("filename", filename)))	1✔
45	defer otelx.End(span, &err)	1✔
46		1✔
47	// We retry the process creation, because it sometimes times out.	1✔
48	const processVMTimeout = 1 * time.Second	1✔
49	return backoff.RetryWithData(func() (_ string, err error) {	2✔
50	ctx, span := tracer.Start(ctx, "jsonnetsecure.ProcessVM.EvaluateAnonymousSnippet.run")	1✔
51	defer otelx.End(span, &err)	1✔
52		1✔
53	ctx, cancel := context.WithTimeout(ctx, processVMTimeout)	1✔
54	defer cancel()	1✔
55		1✔
56	var (	1✔
57	stdin bytes.Buffer	1✔
58	)	1✔
59		1✔
60	p.params.Filename = filename	1✔
61	p.params.Snippet = snippet	1✔
62		1✔
63	if err := p.params.EncodeTo(&stdin); err != nil {	1✔
64	return "", backoff.Permanent(errors.WithStack(err))	×
65	}	×
66
67	cmd := exec.CommandContext(ctx, p.path, p.args...) //nolint:gosec	1✔
68	cmd.Stdin = &stdin	1✔
69	cmd.Env = []string{"GOMAXPROCS=1"}	1✔
70	cmd.WaitDelay = 100 * time.Millisecond	1✔
71		1✔
72	stdoutPipe, err := cmd.StdoutPipe()	1✔
73	if err != nil {	1✔
NEW 74	return "", backoff.Permanent(errors.WithStack(err))	×
UNCOV 75	}	×
76	defer stdoutPipe.Close()	1✔
77		1✔
78	stderrPipe, err := cmd.StderrPipe()	1✔
79	if err != nil {	1✔
NEW 80	return "", backoff.Permanent(errors.WithStack(err))	×
NEW 81	}	×
82	defer stderrPipe.Close()	1✔
83		1✔
84	stdoutReader := io.LimitReader(stdoutPipe, jsonnetOutputLimit)	1✔
85	stderrReader := io.LimitReader(stderrPipe, jsonnetErrLimit)	1✔
86		1✔
87	if err := cmd.Start(); err != nil {	1✔
NEW 88	if errors.Is(err, syscall.EAGAIN) \|\| errors.Is(err, syscall.ENOMEM) {	×
NEW 89	return "", errors.WithStack(fmt.Errorf("jsonnetsecure: failed to start subprocess, retrying %v", err))	×
NEW 90	}	×
NEW 91	return "", backoff.Permanent(fmt.Errorf("jsonnetsecure: failed to start subprocess: %w", err))	×
92	}
93
94	stdoutOutput, err := io.ReadAll(stdoutReader)	1✔
95	if err != nil {	1✔
NEW 96	return "", backoff.Permanent(fmt.Errorf("jsonnetsecure: failed to read subprocess stdout: %w", err))	×
NEW 97	}	×
98
99	// Reading from stderr is best effort (there might not be anything).
100	stderrOutput, _ := io.ReadAll(stderrReader)	1✔
101		1✔
102	// If there was some stderr output or the stdout has reached the limit,	1✔
103	// no point in keeping the subprocess running so we kill it.	1✔
104	// This limits the negative effect of misbehaving jsonnet scripts.	1✔
105	// NOTE: Depending on what the subprocess does and the OS scheduling, this might kill the subprocess, or have no effect (e.g. the child already terminated).	1✔
106	if len(stderrOutput) > 0 \|\| len(stdoutOutput) == int(jsonnetOutputLimit) {	2✔
107	cmd.Cancel()	1✔
108		1✔
109	return "", backoff.Permanent(fmt.Errorf("jsonnetsecure: subprocess encountered an error or reached limits: stderr=%s stdout_length=%d", string(stderrOutput), len(stdoutOutput)))	1✔
110	}	1✔
111
112	err = cmd.Wait()	1✔
113	if err != nil \|\| len(stderrOutput) > 0 {	1✔
NEW 114	return "", backoff.Permanent(fmt.Errorf("jsonnetsecure: subprocess encountered an error: %w %s", err, string(stderrOutput)))	×
UNCOV 115	}	×
116
117	return string(stdoutOutput), nil	1✔
118	}, backoff.WithContext(backoff.NewExponentialBackOff(), ctx))
119	}
120
121	func (p *ProcessVM) ExtCode(key string, val string) {	1✔
122	p.params.ExtCodes = append(p.params.ExtCodes, kv{key, val})	1✔
123	}	1✔
124
125	func (p *ProcessVM) ExtVar(key string, val string) {	1✔
126	p.params.ExtVars = append(p.params.ExtVars, kv{key, val})	1✔
127	}	1✔
128
129	func (p *ProcessVM) TLACode(key string, val string) {	1✔
130	p.params.TLACodes = append(p.params.TLACodes, kv{key, val})	1✔
131	}	1✔
132
133	func (p *ProcessVM) TLAVar(key string, val string) {	1✔
134	p.params.TLAVars = append(p.params.TLAVars, kv{key, val})	1✔
135	}	1✔
136
137	func (pp *processParameters) EncodeTo(w io.Writer) error {	1✔
138	return json.NewEncoder(w).Encode(pp)	1✔
139	}	1✔
140
141	func (pp *processParameters) DecodeFrom(r io.Reader) error {	×
142	return json.NewDecoder(r).Decode(pp)	×
143	}	×
144
145	func (pp *processParameters) Decode(d []byte) error {	×
146	return json.Unmarshal(d, pp)	×
147	}	×

ory / x / 15042063902

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous