5975519062

Committed 25 Aug 2023 11:50AM UTC coverage: 66.778% (+0.07%) from 66.71%

Build # 5975519062

Build Type

push

Committed by

Commit Message

chore(restore): add more logs for restore request (#8050) (#8975)

Run Details

8 of 8 new or added lines in 3 files covered. (100.0%)

58220 of 87184 relevant lines covered (66.78%)

2210219.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

54.55

/dgraph/cmd/zero/http.go

/*
 * Copyright 2023 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package zero

import (
        "context"
        "encoding/json"
        "fmt"
        "net/http"
        "strconv"
        "strings"
        "time"

        "github.com/gogo/protobuf/jsonpb"
        "github.com/golang/glog"
        "github.com/pkg/errors"

        "github.com/dgraph-io/dgo/v230/protos/api"
        "github.com/dgraph-io/dgraph/protos/pb"
        "github.com/dgraph-io/dgraph/x"
)

// intFromQueryParam checks for name as a query param, converts it to uint64 and returns it.
// It also writes any errors to w. A bool is returned to indicate if the param was parsed
// successfully.
func intFromQueryParam(w http.ResponseWriter, r *http.Request, name string) (uint64, bool) {
        str := r.URL.Query().Get(name)
        if len(str) == 0 {
                w.WriteHeader(http.StatusBadRequest)
                x.SetStatus(w, x.ErrorInvalidRequest, fmt.Sprintf("%s not passed", name))
                return 0, false
        }
        val, err := strconv.ParseUint(str, 0, 64)
        if err != nil {
                w.WriteHeader(http.StatusBadRequest)
                x.SetStatus(w, x.ErrorInvalidRequest, fmt.Sprintf("Error while parsing %s", name))
                return 0, false
        }
        return val, true
}

func (st *state) assign(w http.ResponseWriter, r *http.Request) {
        x.AddCorsHeaders(w)
        w.Header().Set("Content-Type", "application/json")
        if r.Method == "OPTIONS" {
                return
        }
        if r.Method != http.MethodGet {
                w.WriteHeader(http.StatusBadRequest)
                x.SetStatus(w, x.ErrorInvalidMethod, "Invalid method")
                return
        }
        val, ok := intFromQueryParam(w, r, "num")
        if !ok {
                return
        }

        num := &pb.Num{Val: val}
        ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
        defer cancel()

        var ids *pb.AssignedIds
        var err error
        what := r.URL.Query().Get("what")
        switch what {
        case "uids":
                num.Type = pb.Num_UID
                ids, err = st.zero.AssignIds(ctx, num)
        case "timestamps":
                num.Type = pb.Num_TXN_TS
                if num.Val == 0 {
                        num.ReadOnly = true
                }
                ids, err = st.zero.Timestamps(ctx, num)
        case "nsids":
                num.Type = pb.Num_NS_ID
                ids, err = st.zero.AssignIds(ctx, num)
        default:
                x.SetStatus(w, x.Error,
                        fmt.Sprintf("Invalid what: [%s]. Must be one of: [uids, timestamps, nsids]", what))
                return
        }
        if err != nil {
                x.SetStatus(w, x.Error, err.Error())
                return
        }

        m := jsonpb.Marshaler{EmitDefaults: true}
        if err := m.Marshal(w, ids); err != nil {
                x.SetStatus(w, x.ErrorNoData, err.Error())
                return
        }
}

// removeNode can be used to remove a node from the cluster. It takes in the RAFT id of the node
// and the group it belongs to. It can be used to remove Dgraph alpha and Zero nodes(group=0).
func (st *state) removeNode(w http.ResponseWriter, r *http.Request) {
        x.AddCorsHeaders(w)
        if r.Method == "OPTIONS" {
                return
        }
        if r.Method != http.MethodGet {
                w.WriteHeader(http.StatusBadRequest)
                x.SetStatus(w, x.ErrorInvalidMethod, "Invalid method")
                return
        }

        nodeId, ok := intFromQueryParam(w, r, "id")
        if !ok {
                return
        }
        groupId, ok := intFromQueryParam(w, r, "group")
        if !ok {
                return
        }

        if _, err := st.zero.RemoveNode(
                context.Background(),
                &pb.RemoveNodeRequest{NodeId: nodeId, GroupId: uint32(groupId)},
        ); err != nil {
                x.SetStatus(w, x.Error, err.Error())
                return
        }
        _, err := fmt.Fprintf(w, "Removed node with group: %v, idx: %v", groupId, nodeId)
        if err != nil {
                glog.Warningf("Error while writing response: %+v", err)
        }
}

// moveTablet can be used to move a tablet to a specific group. It takes in tablet and group as
// argument.
func (st *state) moveTablet(w http.ResponseWriter, r *http.Request) {
        x.AddCorsHeaders(w)
        if r.Method == "OPTIONS" {
                return
        }
        if r.Method != http.MethodGet {
                w.WriteHeader(http.StatusBadRequest)
                x.SetStatus(w, x.ErrorInvalidMethod, "Invalid method")
                return
        }

        if !st.node.AmLeader() {
                w.WriteHeader(http.StatusBadRequest)
                x.SetStatus(w, x.ErrorInvalidRequest,
                        "This Zero server is not the leader. Re-run command on leader.")
                return
        }

        namespace := r.URL.Query().Get("namespace")
        namespace = strings.TrimSpace(namespace)
        ns := x.GalaxyNamespace
        if namespace != "" {
                var err error
                if ns, err = strconv.ParseUint(namespace, 0, 64); err != nil {
                        w.WriteHeader(http.StatusBadRequest)
                        x.SetStatus(w, x.ErrorInvalidRequest, "Invalid namespace in query parameter.")
                        return
                }
        }

        tablet := r.URL.Query().Get("tablet")
        if len(tablet) == 0 {
                w.WriteHeader(http.StatusBadRequest)
                x.SetStatus(w, x.ErrorInvalidRequest, "tablet is a mandatory query parameter")
                return
        }

        groupId, ok := intFromQueryParam(w, r, "group")
        if !ok {
                w.WriteHeader(http.StatusBadRequest)
                x.SetStatus(w, x.ErrorInvalidRequest,
                        "Query parameter 'group' should contain a valid integer.")
                return
        }
        dstGroup := uint32(groupId)

        var resp *pb.Status
        var err error
        if resp, err = st.zero.MoveTablet(
                context.Background(),
                &pb.MoveTabletRequest{Namespace: ns, Tablet: tablet, DstGroup: dstGroup},
        ); err != nil {
                if resp.GetMsg() == x.ErrorInvalidRequest {
                        w.WriteHeader(http.StatusBadRequest)
                        x.SetStatus(w, x.ErrorInvalidRequest, err.Error())
                } else {
                        w.WriteHeader(http.StatusInternalServerError)
                        x.SetStatus(w, x.Error, err.Error())
                }
                return
        }
        _, err = fmt.Fprint(w, resp.GetMsg())
        if err != nil {
                glog.Warningf("Error while writing response: %+v", err)
        }
}

func (st *state) getState(w http.ResponseWriter, r *http.Request) {
        x.AddCorsHeaders(w)
        w.Header().Set("Content-Type", "application/json")

        ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
        defer cancel()
        if err := st.node.WaitLinearizableRead(ctx); err != nil {
                w.WriteHeader(http.StatusInternalServerError)
                x.SetStatus(w, x.Error, err.Error())
                return
        }
        mstate := st.zero.membershipState()
        if mstate == nil {
                x.SetStatus(w, x.ErrorNoData, "No membership state found.")
                return
        }

        m := jsonpb.Marshaler{EmitDefaults: true}
        if err := m.Marshal(w, mstate); err != nil {
                x.SetStatus(w, x.ErrorNoData, err.Error())
                return
        }
}

func (s *Server) zeroHealth(ctx context.Context) (*api.Response, error) {
        if ctx.Err() != nil {
                return nil, errors.Wrap(ctx.Err(), "http request context error")
        }
        health := pb.HealthInfo{
                Instance: "zero",
                Address:  x.WorkerConfig.MyAddr,
                Status:   "healthy",
                Version:  x.Version(),
                Uptime:   int64(time.Since(x.WorkerConfig.StartTime) / time.Second),
                LastEcho: time.Now().Unix(),
        }
        jsonOut, err := json.Marshal(health)
        if err != nil {
                return nil, errors.Wrapf(err, "unable to marshal zero health, error")
        }
        return &api.Response{Json: jsonOut}, nil
}

func (st *state) pingResponse(w http.ResponseWriter, r *http.Request) {
        x.AddCorsHeaders(w)

        /*
         * zero is changed to also output the health in JSON format for client
         * request header "Accept: application/json".
         *
         * Backward compatibility- Before this change the '/health' endpoint
         * used to output the string OK. After the fix it returns OK when the
         * client sends the request without "Accept: application/json" in its
         * http header.
         */
        switch r.Header.Get("Accept") {
        case "application/json":
                resp, err := (st.zero).zeroHealth(r.Context())
                if err != nil {
                        x.SetStatus(w, x.Error, err.Error())
                        return
                }
                w.Header().Set("Content-Type", "application/json; charset=utf-8")
                w.WriteHeader(http.StatusOK)
                if _, err := w.Write(resp.Json); err != nil {
                        glog.Warningf("http error send failed, error msg=[%v]", err)
                        return
                }
        default:
                w.Header().Set("Content-Type", "text/plain; charset=utf-8")
                w.WriteHeader(http.StatusOK)
                if _, err := w.Write([]byte("OK")); err != nil {
                        glog.Warningf("http error send failed, error msg=[%v]", err)
                        return
                }
        }
}

1	/*
2	* Copyright 2023 Dgraph Labs, Inc. and Contributors
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package zero
18
19	import (
20	"context"
21	"encoding/json"
22	"fmt"
23	"net/http"
24	"strconv"
25	"strings"
26	"time"
27
28	"github.com/gogo/protobuf/jsonpb"
29	"github.com/golang/glog"
30	"github.com/pkg/errors"
31
32	"github.com/dgraph-io/dgo/v230/protos/api"
33	"github.com/dgraph-io/dgraph/protos/pb"
34	"github.com/dgraph-io/dgraph/x"
35	)
36
37	// intFromQueryParam checks for name as a query param, converts it to uint64 and returns it.
38	// It also writes any errors to w. A bool is returned to indicate if the param was parsed
39	// successfully.
40	func intFromQueryParam(w http.ResponseWriter, r *http.Request, name string) (uint64, bool) {	4,013✔
41	str := r.URL.Query().Get(name)	4,013✔
42	if len(str) == 0 {	4,013✔
43	w.WriteHeader(http.StatusBadRequest)	×
44	x.SetStatus(w, x.ErrorInvalidRequest, fmt.Sprintf("%s not passed", name))	×
45	return 0, false	×
46	}	×
47	val, err := strconv.ParseUint(str, 0, 64)	4,013✔
48	if err != nil {	4,013✔
49	w.WriteHeader(http.StatusBadRequest)	×
50	x.SetStatus(w, x.ErrorInvalidRequest, fmt.Sprintf("Error while parsing %s", name))	×
51	return 0, false	×
52	}	×
53	return val, true	4,013✔
54	}
55
56	func (st state) assign(w http.ResponseWriter, r http.Request) {	1,005✔
57	x.AddCorsHeaders(w)	1,005✔
58	w.Header().Set("Content-Type", "application/json")	1,005✔
59	if r.Method == "OPTIONS" {	1,005✔
60	return	×
61	}	×
62	if r.Method != http.MethodGet {	1,005✔
63	w.WriteHeader(http.StatusBadRequest)	×
64	x.SetStatus(w, x.ErrorInvalidMethod, "Invalid method")	×
65	return	×
66	}	×
67	val, ok := intFromQueryParam(w, r, "num")	1,005✔
68	if !ok {	1,005✔
69	return	×
70	}	×
71
72	num := &pb.Num{Val: val}	1,005✔
73	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)	1,005✔
74	defer cancel()	1,005✔
75		1,005✔
76	var ids *pb.AssignedIds	1,005✔
77	var err error	1,005✔
78	what := r.URL.Query().Get("what")	1,005✔
79	switch what {	1,005✔
80	case "uids":	1,005✔
81	num.Type = pb.Num_UID	1,005✔
82	ids, err = st.zero.AssignIds(ctx, num)	1,005✔
83	case "timestamps":	×
84	num.Type = pb.Num_TXN_TS	×
85	if num.Val == 0 {	×
86	num.ReadOnly = true	×
87	}	×
88	ids, err = st.zero.Timestamps(ctx, num)	×
89	case "nsids":	×
90	num.Type = pb.Num_NS_ID	×
91	ids, err = st.zero.AssignIds(ctx, num)	×
92	default:	×
93	x.SetStatus(w, x.Error,	×
94	fmt.Sprintf("Invalid what: [%s]. Must be one of: [uids, timestamps, nsids]", what))	×
95	return	×
96	}
97	if err != nil {	1,006✔
98	x.SetStatus(w, x.Error, err.Error())	1✔
99	return	1✔
100	}	1✔
101
102	m := jsonpb.Marshaler{EmitDefaults: true}	1,004✔
103	if err := m.Marshal(w, ids); err != nil {	1,004✔
104	x.SetStatus(w, x.ErrorNoData, err.Error())	×
105	return	×
106	}	×
107	}
108
109	// removeNode can be used to remove a node from the cluster. It takes in the RAFT id of the node
110	// and the group it belongs to. It can be used to remove Dgraph alpha and Zero nodes(group=0).
111	func (st state) removeNode(w http.ResponseWriter, r http.Request) {	1,000✔
112	x.AddCorsHeaders(w)	1,000✔
113	if r.Method == "OPTIONS" {	1,000✔
114	return	×
115	}	×
116	if r.Method != http.MethodGet {	1,000✔
117	w.WriteHeader(http.StatusBadRequest)	×
118	x.SetStatus(w, x.ErrorInvalidMethod, "Invalid method")	×
119	return	×
120	}	×
121
122	nodeId, ok := intFromQueryParam(w, r, "id")	1,000✔
123	if !ok {	1,000✔
124	return	×
125	}	×
126	groupId, ok := intFromQueryParam(w, r, "group")	1,000✔
127	if !ok {	1,000✔
128	return	×
129	}	×
130
131	if _, err := st.zero.RemoveNode(	1,000✔
132	context.Background(),	1,000✔
133	&pb.RemoveNodeRequest{NodeId: nodeId, GroupId: uint32(groupId)},	1,000✔
134	); err != nil {	2,000✔
135	x.SetStatus(w, x.Error, err.Error())	1,000✔
136	return	1,000✔
137	}	1,000✔
138	_, err := fmt.Fprintf(w, "Removed node with group: %v, idx: %v", groupId, nodeId)	×
139	if err != nil {	×
140	glog.Warningf("Error while writing response: %+v", err)	×
141	}	×
142	}
143
144	// moveTablet can be used to move a tablet to a specific group. It takes in tablet and group as
145	// argument.
146	func (st state) moveTablet(w http.ResponseWriter, r http.Request) {	1,008✔
147	x.AddCorsHeaders(w)	1,008✔
148	if r.Method == "OPTIONS" {	1,008✔
149	return	×
150	}	×
151	if r.Method != http.MethodGet {	1,008✔
152	w.WriteHeader(http.StatusBadRequest)	×
153	x.SetStatus(w, x.ErrorInvalidMethod, "Invalid method")	×
154	return	×
155	}	×
156
157	if !st.node.AmLeader() {	1,008✔
158	w.WriteHeader(http.StatusBadRequest)	×
159	x.SetStatus(w, x.ErrorInvalidRequest,	×
160	"This Zero server is not the leader. Re-run command on leader.")	×
161	return	×
162	}	×
163
164	namespace := r.URL.Query().Get("namespace")	1,008✔
165	namespace = strings.TrimSpace(namespace)	1,008✔
166	ns := x.GalaxyNamespace	1,008✔
167	if namespace != "" {	1,008✔
168	var err error	×
169	if ns, err = strconv.ParseUint(namespace, 0, 64); err != nil {	×
170	w.WriteHeader(http.StatusBadRequest)	×
171	x.SetStatus(w, x.ErrorInvalidRequest, "Invalid namespace in query parameter.")	×
172	return	×
173	}	×
174	}
175
176	tablet := r.URL.Query().Get("tablet")	1,008✔
177	if len(tablet) == 0 {	1,008✔
178	w.WriteHeader(http.StatusBadRequest)	×
179	x.SetStatus(w, x.ErrorInvalidRequest, "tablet is a mandatory query parameter")	×
180	return	×
181	}	×
182
183	groupId, ok := intFromQueryParam(w, r, "group")	1,008✔
184	if !ok {	1,008✔
185	w.WriteHeader(http.StatusBadRequest)	×
186	x.SetStatus(w, x.ErrorInvalidRequest,	×
187	"Query parameter 'group' should contain a valid integer.")	×
188	return	×
189	}	×
190	dstGroup := uint32(groupId)	1,008✔
191		1,008✔
192	var resp *pb.Status	1,008✔
193	var err error	1,008✔
194	if resp, err = st.zero.MoveTablet(	1,008✔
195	context.Background(),	1,008✔
196	&pb.MoveTabletRequest{Namespace: ns, Tablet: tablet, DstGroup: dstGroup},	1,008✔
197	); err != nil {	2,011✔
198	if resp.GetMsg() == x.ErrorInvalidRequest {	2,006✔
199	w.WriteHeader(http.StatusBadRequest)	1,003✔
200	x.SetStatus(w, x.ErrorInvalidRequest, err.Error())	1,003✔
201	} else {	1,003✔
202	w.WriteHeader(http.StatusInternalServerError)	×
203	x.SetStatus(w, x.Error, err.Error())	×
204	}	×
205	return	1,003✔
206	}
207	_, err = fmt.Fprint(w, resp.GetMsg())	5✔
208	if err != nil {	5✔
209	glog.Warningf("Error while writing response: %+v", err)	×
210	}	×
211	}
212
213	func (st state) getState(w http.ResponseWriter, r http.Request) {	164✔
214	x.AddCorsHeaders(w)	164✔
215	w.Header().Set("Content-Type", "application/json")	164✔
216		164✔
217	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)	164✔
218	defer cancel()	164✔
219	if err := st.node.WaitLinearizableRead(ctx); err != nil {	164✔
220	w.WriteHeader(http.StatusInternalServerError)	×
221	x.SetStatus(w, x.Error, err.Error())	×
222	return	×
223	}	×
224	mstate := st.zero.membershipState()	164✔
225	if mstate == nil {	164✔
226	x.SetStatus(w, x.ErrorNoData, "No membership state found.")	×
227	return	×
228	}	×
229
230	m := jsonpb.Marshaler{EmitDefaults: true}	164✔
231	if err := m.Marshal(w, mstate); err != nil {	164✔
232	x.SetStatus(w, x.ErrorNoData, err.Error())	×
233	return	×
234	}	×
235	}
236
237	func (s Server) zeroHealth(ctx context.Context) (api.Response, error) {	1✔
238	if ctx.Err() != nil {	1✔
239	return nil, errors.Wrap(ctx.Err(), "http request context error")	×
240	}	×
241	health := pb.HealthInfo{	1✔
242	Instance: "zero",	1✔
243	Address: x.WorkerConfig.MyAddr,	1✔
244	Status: "healthy",	1✔
245	Version: x.Version(),	1✔
246	Uptime: int64(time.Since(x.WorkerConfig.StartTime) / time.Second),	1✔
247	LastEcho: time.Now().Unix(),	1✔
248	}	1✔
249	jsonOut, err := json.Marshal(health)	1✔
250	if err != nil {	1✔
251	return nil, errors.Wrapf(err, "unable to marshal zero health, error")	×
252	}	×
253	return &api.Response{Json: jsonOut}, nil	1✔
254	}
255
256	func (st state) pingResponse(w http.ResponseWriter, r http.Request) {	64✔
257	x.AddCorsHeaders(w)	64✔
258		64✔
259	/*	64✔
260	* zero is changed to also output the health in JSON format for client	64✔
261	* request header "Accept: application/json".	64✔
262	*	64✔
263	* Backward compatibility- Before this change the '/health' endpoint	64✔
264	* used to output the string OK. After the fix it returns OK when the	64✔
265	* client sends the request without "Accept: application/json" in its	64✔
266	* http header.	64✔
267	*/	64✔
268	switch r.Header.Get("Accept") {	64✔
269	case "application/json":	1✔
270	resp, err := (st.zero).zeroHealth(r.Context())	1✔
271	if err != nil {	1✔
272	x.SetStatus(w, x.Error, err.Error())	×
273	return	×
274	}	×
275	w.Header().Set("Content-Type", "application/json; charset=utf-8")	1✔
276	w.WriteHeader(http.StatusOK)	1✔
277	if _, err := w.Write(resp.Json); err != nil {	1✔
278	glog.Warningf("http error send failed, error msg=[%v]", err)	×
279	return	×
280	}	×
281	default:	63✔
282	w.Header().Set("Content-Type", "text/plain; charset=utf-8")	63✔
283	w.WriteHeader(http.StatusOK)	63✔
284	if _, err := w.Write([]byte("OK")); err != nil {	63✔
285	glog.Warningf("http error send failed, error msg=[%v]", err)	×
286	return	×
287	}	×
288	}
289	}

dgraph-io / dgraph / 5975519062

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous