6180409240

Committed 14 Sep 2023 03:02AM UTC coverage: 26.069% (-3.9%) from 29.927%

Build # 6180409240

Build Type

push

github

Committed by

web-flow

Commit Message

feat: Support BYOC (#171)

Signed-off-by: xieydd <xieydd@gmail.com>

Run Details

443 of 443 new or added lines in 14 files covered. (100.0%)

939 of 3602 relevant lines covered (26.07%)

1.63 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/agent/pkg/server/handler_streamlit_proxy.go

package server

import (
        "fmt"
        "io"
        "net"
        "net/http"
        "net/http/httputil"
        "net/url"
        "path"
        "strconv"
        "time"

        "github.com/gin-gonic/gin"
        "github.com/sirupsen/logrus"
        "github.com/tensorchord/openmodelz/agent/pkg/consts"
        "github.com/tensorchord/openmodelz/agent/pkg/server/static"
)

// @Summary     Reverse proxy to streamlit.
// @Description Reverse proxy to streamlit.
// @Tags        inference
// @Accept      */*
// @Produce     json
// @Param       id path string true "Deployment ID"
// @Router      /streamlit/{id} [get]
// @Router      /streamlit/{id} [post]
// @Success     201
func (s *Server) proxyStreamlit(c *gin.Context) error {
        remote, err := url.Parse(fmt.Sprintf("http://0.0.0.0:%d", s.config.Server.ServerPort))
        if err != nil {
                return err
        }
        proxy := httputil.NewSingleHostReverseProxy(remote)
        proxy.Transport = &http.Transport{
                Proxy: http.ProxyFromEnvironment,
                DialContext: (&net.Dialer{
                        Timeout:   s.config.ModelZCloud.UpstreamTimeout,
                        KeepAlive: s.config.ModelZCloud.UpstreamTimeout,
                        DualStack: true,
                }).DialContext,
                MaxIdleConns:          s.config.ModelZCloud.MaxIdleConnections,
                MaxIdleConnsPerHost:   s.config.ModelZCloud.MaxIdleConnectionsPerHost,
                IdleConnTimeout:       90 * time.Second,
                TLSHandshakeTimeout:   10 * time.Second,
                ExpectContinueTimeout: 1 * time.Second,
        }

        uid, deployment, err := s.proxyNoAuth(c)
        if err != nil {
                return err
        }

        ns := consts.DefaultPrefix + uid
        proxy.Director = func(req *http.Request) {
                req.Header = c.Request.Header
                req.Host = remote.Host
                req.URL.Scheme = remote.Scheme
                req.URL.Host = remote.Host
                req.URL.Path = path.Join(
                        "/", "inference", fmt.Sprintf("%s.%s", deployment, ns), c.Param("proxyPath"))

                logrus.WithFields(logrus.Fields{
                        "deployment": deployment,
                        "uid":        uid,
                        "ns":         ns,
                        "path":       req.URL.Path,
                        "remote":     remote.String(),
                }).Debug("proxying to streamlit")
        }
        proxy.ModifyResponse = func(resp *http.Response) error {
                if resp.StatusCode == http.StatusSeeOther {
                        resp.StatusCode = http.StatusOK
                        instances, err := s.runtime.InferenceInstanceList(ns, deployment)
                        if err != nil {
                                return NewError(http.StatusInternalServerError, err, "instance-list")
                        }

                        buf, err := static.RenderDeploymentLoadingPage("streamlit", resp.Header.Get("X-Call-Id"),
                                "We are currently processing your request.", deployment, instances)
                        if err != nil {
                                return NewError(http.StatusInternalServerError, err, "render-loading-page")
                        }
                        resp.Body = io.NopCloser(buf)
                        resp.ContentLength = int64(buf.Len())
                        resp.Header.Set("Content-Length", strconv.Itoa(buf.Len()))
                        resp.Header.Set("Content-Type", "text/html")
                        resp.StatusCode = http.StatusServiceUnavailable
                }
                return nil
        }

        proxy.ServeHTTP(c.Writer, c.Request)
        return nil
}

1	package server
2
3	import (
4	"fmt"
5	"io"
6	"net"
7	"net/http"
8	"net/http/httputil"
9	"net/url"
10	"path"
11	"strconv"
12	"time"
13
14	"github.com/gin-gonic/gin"
15	"github.com/sirupsen/logrus"
16	"github.com/tensorchord/openmodelz/agent/pkg/consts"
17	"github.com/tensorchord/openmodelz/agent/pkg/server/static"
18	)
19
20	// @Summary Reverse proxy to streamlit.
21	// @Description Reverse proxy to streamlit.
22	// @Tags inference
23	// @Accept /
24	// @Produce json
25	// @Param id path string true "Deployment ID"
26	// @Router /streamlit/{id} [get]
27	// @Router /streamlit/{id} [post]
28	// @Success 201
29	func (s Server) proxyStreamlit(c gin.Context) error {	×
30	remote, err := url.Parse(fmt.Sprintf("http://0.0.0.0:%d", s.config.Server.ServerPort))	×
31	if err != nil {	×
32	return err	×
33	}	×
34	proxy := httputil.NewSingleHostReverseProxy(remote)	×
35	proxy.Transport = &http.Transport{	×
36	Proxy: http.ProxyFromEnvironment,	×
37	DialContext: (&net.Dialer{	×
38	Timeout: s.config.ModelZCloud.UpstreamTimeout,	×
39	KeepAlive: s.config.ModelZCloud.UpstreamTimeout,	×
40	DualStack: true,	×
41	}).DialContext,	×
42	MaxIdleConns: s.config.ModelZCloud.MaxIdleConnections,	×
43	MaxIdleConnsPerHost: s.config.ModelZCloud.MaxIdleConnectionsPerHost,	×
44	IdleConnTimeout: 90 * time.Second,	×
45	TLSHandshakeTimeout: 10 * time.Second,	×
46	ExpectContinueTimeout: 1 * time.Second,	×
47	}	×
48		×
49	uid, deployment, err := s.proxyNoAuth(c)	×
50	if err != nil {	×
51	return err	×
52	}	×
53
54	ns := consts.DefaultPrefix + uid	×
55	proxy.Director = func(req *http.Request) {	×
56	req.Header = c.Request.Header	×
57	req.Host = remote.Host	×
58	req.URL.Scheme = remote.Scheme	×
59	req.URL.Host = remote.Host	×
60	req.URL.Path = path.Join(	×
61	"/", "inference", fmt.Sprintf("%s.%s", deployment, ns), c.Param("proxyPath"))	×
62		×
63	logrus.WithFields(logrus.Fields{	×
64	"deployment": deployment,	×
65	"uid": uid,	×
66	"ns": ns,	×
67	"path": req.URL.Path,	×
68	"remote": remote.String(),	×
69	}).Debug("proxying to streamlit")	×
70	}	×
71	proxy.ModifyResponse = func(resp *http.Response) error {	×
72	if resp.StatusCode == http.StatusSeeOther {	×
73	resp.StatusCode = http.StatusOK	×
74	instances, err := s.runtime.InferenceInstanceList(ns, deployment)	×
75	if err != nil {	×
76	return NewError(http.StatusInternalServerError, err, "instance-list")	×
77	}	×
78
79	buf, err := static.RenderDeploymentLoadingPage("streamlit", resp.Header.Get("X-Call-Id"),	×
80	"We are currently processing your request.", deployment, instances)	×
81	if err != nil {	×
82	return NewError(http.StatusInternalServerError, err, "render-loading-page")	×
83	}	×
84	resp.Body = io.NopCloser(buf)	×
85	resp.ContentLength = int64(buf.Len())	×
86	resp.Header.Set("Content-Length", strconv.Itoa(buf.Len()))	×
87	resp.Header.Set("Content-Type", "text/html")	×
88	resp.StatusCode = http.StatusServiceUnavailable	×
89	}
90	return nil	×
91	}
92
93	proxy.ServeHTTP(c.Writer, c.Request)	×
94	return nil	×
95	}

tensorchord / openmodelz / 6180409240

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous