16273924685

Committed 14 Jul 2025 05:44PM UTC coverage: 89.803% (+0.02%) from 89.781%

Build # 16273924685

Build Type

Pull #76

github

Committed by

robertodauria

Commit Message

Move counter increase out of else block

Pull Request Pull Request #76: Add Prometheus metrics for garbage collector operations

Run Details

6 of 8 new or added lines in 1 file covered. (75.0%)

5 existing lines in 1 file now uncovered.

1321 of 1471 relevant lines covered (89.8%)

0.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.18

/handler/handler.go

package handler

import (
        "context"
        "encoding/json"
        "errors"
        "fmt"
        "log"
        "net"
        "net/http"
        "regexp"
        "strconv"
        "strings"

        "github.com/Masterminds/semver/v3"
        v0 "github.com/m-lab/autojoin/api/v0"
        "github.com/m-lab/autojoin/iata"
        "github.com/m-lab/autojoin/internal/adminx"
        "github.com/m-lab/autojoin/internal/dnsname"
        "github.com/m-lab/autojoin/internal/dnsx"
        "github.com/m-lab/autojoin/internal/dnsx/dnsiface"
        "github.com/m-lab/autojoin/internal/register"
        "github.com/m-lab/gcp-service-discovery/discovery"
        "github.com/m-lab/go/host"
        "github.com/m-lab/go/rtx"
        v2 "github.com/m-lab/locate/api/v2"
        "github.com/m-lab/uuid-annotator/annotator"
        "github.com/oschwald/geoip2-golang"
)

var (
        errLocationNotFound = errors.New("location not found")
        errLocationFormat   = errors.New("location could not be parsed")

        validName = regexp.MustCompile(`[a-zA-Z0-9]+`)
)

// Server maintains shared state for the server.
type Server struct {
        Project    string
        Iata       IataFinder
        Maxmind    MaxmindFinder
        ASN        ASNFinder
        DNS        dnsiface.Service
        minVersion *semver.Version

        sm         ServiceAccountSecretManager
        dnsTracker DNSTracker
        dsm        Datastore
}

// ASNFinder is an interface used by the Server to manage ASN information.
type ASNFinder interface {
        AnnotateIP(src string) *annotator.Network
        Reload(ctx context.Context)
}

// MaxmindFinder is an interface used by the Server to manage Maxmind information.
type MaxmindFinder interface {
        City(ip net.IP) (*geoip2.City, error)
        Reload(ctx context.Context) error
}

// IataFinder is an interface used by the Server to manage IATA information.
type IataFinder interface {
        Lookup(country string, lat, lon float64) (string, error)
        Find(iata string) (iata.Row, error)
        Load(ctx context.Context) error
}

type DNSTracker interface {
        Update(string, []string) error
        Delete(string) error
        List() ([]string, [][]string, error)
}

// ServiceAccountSecretManager is an interface used by the server to allocate service account keys.
type ServiceAccountSecretManager interface {
        LoadOrCreateKey(ctx context.Context, org string) (string, error)
}

type Datastore interface {
        GetOrganization(ctx context.Context, name string) (*adminx.Organization, error)
}

// NewServer creates a new Server instance for request handling.
func NewServer(project string, finder IataFinder, maxmind MaxmindFinder, asn ASNFinder,
        ds dnsiface.Service, tracker DNSTracker, sm ServiceAccountSecretManager, dsm Datastore,
        minVersion string) *Server {
        v, err := semver.NewVersion(minVersion)
        rtx.Must(err, "invalid minimum version")
        return &Server{
                Project:    project,
                Iata:       finder,
                Maxmind:    maxmind,
                ASN:        asn,
                DNS:        ds,
                sm:         sm,
                minVersion: v,

                dnsTracker: tracker,
                dsm:        dsm,
        }
}

// Reload reloads all resources used by the Server.
func (s *Server) Reload(ctx context.Context) {
        s.Iata.Load(ctx)
        s.Maxmind.Reload(ctx)
}

// Lookup is a handler used to find the nearest IATA given client IP or lat/lon metadata.
func (s *Server) Lookup(rw http.ResponseWriter, req *http.Request) {
        resp := v0.LookupResponse{}
        country, err := s.getCountry(req)
        if country == "" || err != nil {
                resp.Error = &v2.Error{
                        Type:   "?country=<country>",
                        Title:  "could not determine country from request",
                        Status: http.StatusBadRequest,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        lat, lon, err := s.getLocation(req)
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "?lat=<lat>&lon=<lon>",
                        Title:  "could not determine lat/lon from request",
                        Status: http.StatusBadRequest,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        code, err := s.Iata.Lookup(country, lat, lon)
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "internal error",
                        Title:  "could not determine iata from request",
                        Status: http.StatusInternalServerError,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        resp.Lookup = &v0.Lookup{
                IATA: code,
        }
        writeResponse(rw, resp)
}

// Register handler is used by autonodes to register their hostname with M-Lab
// on startup and receive additional needed configuration metadata.
func (s *Server) Register(rw http.ResponseWriter, req *http.Request) {
        // All replies, errors and successes, should be json.
        rw.Header().Set("Content-Type", "application/json")

        resp := v0.RegisterResponse{}

        // Check version first.
        versionStr := req.URL.Query().Get("version")
        // If no version is provided, default to v0.0.0. This allows existing clients
        // that do not provide the version yet to keep working until a minVersion is set.
        if versionStr == "" {
                versionStr = "v0.0.0"
        }

        // Parse the provided version.
        clientVersion, err := semver.NewVersion(versionStr)
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "version.invalid",
                        Title:  "invalid version format - must be semantic version (e.g. v1.2.3)",
                        Detail: err.Error(),
                        Status: http.StatusBadRequest,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }

        if clientVersion.LessThan(s.minVersion) {
                resp.Error = &v2.Error{
                        Type: "version.outdated",
                        Title: fmt.Sprintf("version %s is below minimum required version %s",
                                clientVersion.String(), s.minVersion.String()),
                        Status: http.StatusForbidden,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }

        param := &register.Params{Project: s.Project}
        param.Service = req.URL.Query().Get("service")
        if !isValidName(param.Service) {
                resp.Error = &v2.Error{
                        Type:   "?service=<service>",
                        Title:  "could not determine service from request",
                        Status: http.StatusBadRequest,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }

        // Get the organization from the context.
        org, ok := req.Context().Value(orgContextKey).(string)
        if !ok {
                resp.Error = &v2.Error{
                        Type:   "auth.context",
                        Title:  "missing organization in context",
                        Status: http.StatusInternalServerError,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        param.Org = org
        param.IPv6 = checkIP(req.URL.Query().Get("ipv6")) // optional.
        param.IPv4 = checkIP(getClientIP(req))
        ip := net.ParseIP(param.IPv4)
        if ip == nil || ip.To4() == nil {
                resp.Error = &v2.Error{
                        Type:   "?ipv4=<ipv4>",
                        Title:  "could not determine client ipv4 from request",
                        Status: http.StatusBadRequest,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        param.Type = req.URL.Query().Get("type")
        if !isValidType(param.Type) {
                resp.Error = &v2.Error{
                        Type:   "?type=<type>",
                        Title:  "invalid machine type from request",
                        Status: http.StatusBadRequest,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        param.Uplink = req.URL.Query().Get("uplink")
        if !isValidUplink(param.Uplink) {
                resp.Error = &v2.Error{
                        Type:   "?uplink=<uplink>",
                        Title:  "invalid uplink speed from request",
                        Status: http.StatusBadRequest,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        iata := getClientIata(req)
        if iata == "" {
                resp.Error = &v2.Error{
                        Type:   "?iata=<iata>",
                        Title:  "could not determine iata from request",
                        Status: http.StatusBadRequest,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        row, err := s.Iata.Find(iata)
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "iata.find",
                        Title:  "could not find given iata in dataset",
                        Status: http.StatusInternalServerError,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        param.Metro = row
        record, err := s.Maxmind.City(ip)
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "maxmind.city",
                        Title:  "could not find city metadata from ip",
                        Status: http.StatusInternalServerError,
                }
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        param.Geo = record
        param.Network = s.ASN.AnnotateIP(param.IPv4)

        // Get the organization probability multiplier.
        orgEntity, err := s.dsm.GetOrganization(req.Context(), param.Org)
        orgMultiplier := 1.0
        if err == nil && orgEntity != nil && orgEntity.ProbabilityMultiplier != nil {
                orgMultiplier = *orgEntity.ProbabilityMultiplier
        }
        // Assign the probability by multiplying the org multiplier with the
        // probability requested by the client.
        param.Probability = getProbability(req) * orgMultiplier
        r := register.CreateRegisterResponse(param)

        key, err := s.sm.LoadOrCreateKey(req.Context(), param.Org)
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "load.serviceaccount.key",
                        Title:  "could not load service account key for node",
                        Status: http.StatusInternalServerError,
                }
                log.Println("loading service account key failure:", err)
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }
        r.Registration.Credentials = &v0.Credentials{
                ServiceAccountKey: key,
        }

        // Register the hostname under the organization zone.
        m := dnsx.NewManager(s.DNS, s.Project, dnsname.OrgZone(param.Org, s.Project))
        _, err = m.Register(req.Context(), r.Registration.Hostname+".", param.IPv4, param.IPv6)
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "dns.register",
                        Title:  "could not register dynamic hostname",
                        Status: http.StatusInternalServerError,
                }
                log.Println("dns register failure:", err)
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }

        // Add the hostname to the DNS tracker.
        err = s.dnsTracker.Update(r.Registration.Hostname, getPorts(req))
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "tracker.gc",
                        Title:  "could not update DNS tracker",
                        Status: http.StatusInternalServerError,
                }
                log.Println("dns gc update failure:", err)
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }

        b, _ := json.MarshalIndent(r, "", " ")
        rw.Write(b)
}

// Delete handler is used by operators to delete a previously registered
// hostname from DNS.
func (s *Server) Delete(rw http.ResponseWriter, req *http.Request) {
        // All replies, errors and successes, should be json.
        rw.Header().Set("Content-Type", "application/json")

        resp := v0.DeleteResponse{}
        hostname := req.URL.Query().Get("hostname")
        name, err := host.Parse(hostname)
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "dns.delete",
                        Title:  "failed to parse hostname",
                        Detail: err.Error(),
                        Status: http.StatusBadRequest,
                }
                log.Println("dns delete (parse) failure:", err)
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }

        m := dnsx.NewManager(s.DNS, s.Project, dnsname.OrgZone(name.Org, s.Project))
        _, err = m.Delete(req.Context(), name.StringAll()+".")
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "dns.delete",
                        Title:  "failed to delete hostname",
                        Detail: err.Error(),
                        Status: http.StatusInternalServerError,
                }
                log.Println("dns delete failure:", err)
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }

        err = s.dnsTracker.Delete(name.StringAll())
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "tracker.gc",
                        Title:  "failed to delete hostname from DNS tracker",
                        Detail: err.Error(),
                        Status: http.StatusInternalServerError,
                }
                log.Println("dns gc delete failure:", err)
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }

        b, err := json.MarshalIndent(resp, "", " ")
        rtx.Must(err, "failed to marshal DNS delete response")
        rw.Write(b)
}

// List handler is used by monitoring to generate a list of known, active
// hostnames previously registered with the Autojoin API.
func (s *Server) List(rw http.ResponseWriter, req *http.Request) {
        // Set CORS policy to allow third-party websites to use returned resources.
        rw.Header().Set("Content-Type", "application/json")
        rw.Header().Set("Access-Control-Allow-Origin", "*")
        rw.Header().Set("Cache-Control", "no-store") // Prevent caching of result.

        configs := []discovery.StaticConfig{}
        resp := v0.ListResponse{}
        hosts, ports, err := s.dnsTracker.List()
        if err != nil {
                resp.Error = &v2.Error{
                        Type:   "list",
                        Title:  "failed to list node records",
                        Detail: err.Error(),
                        Status: http.StatusInternalServerError,
                }
                log.Println("list failure:", err)
                rw.WriteHeader(resp.Error.Status)
                writeResponse(rw, resp)
                return
        }

        org := req.URL.Query().Get("org")
        format := req.URL.Query().Get("format")
        sites := map[string]bool{}

        // Create a prometheus StaticConfig for each known host.
        for i := range hosts {
                h, err := host.Parse(hosts[i])
                if err != nil {
                        continue
                }
                if org != "" && org != h.Org {
                        // Skip hosts that are not part of the given org.
                        continue
                }
                sites[h.Site] = true
                if format == "script-exporter" {
                        // NOTE: do not assign any ports for script exporter.
                        ports[i] = []string{""}
                } else {
                        // Convert port strings to ":<port>".
                        p := []string{}
                        for j := range ports[i] {
                                p = append(p, ":"+ports[i][j])
                        }
                        ports[i] = p
                }
                for _, port := range ports[i] {
                        labels := map[string]string{
                                "machine":    hosts[i],
                                "type":       "virtual",
                                "deployment": "byos",
                                "managed":    "none",
                                "org":        h.Org,
                        }
                        if req.URL.Query().Get("service") != "" {
                                labels["service"] = req.URL.Query().Get("service")
                        }
                        // We create one record per host to add a unique "machine" label to each one.
                        configs = append(configs, discovery.StaticConfig{
                                Targets: []string{hosts[i] + port},
                                Labels:  labels,
                        })
                }
        }

        var results interface{}
        switch format {
        case "script-exporter":
                fallthrough
        case "blackbox":
                fallthrough
        case "prometheus":
                results = configs
        case "servers":
                resp.Servers = hosts
                results = resp
        case "sites":
                for k := range sites {
                        resp.Sites = append(resp.Sites, k)
                }
                results = resp
        default:
                resp.Servers = hosts
                results = resp
        }
        // Generate as JSON; the list may be empty.
        b, err := json.MarshalIndent(results, "", " ")
        rtx.Must(err, "failed to marshal DNS delete response")
        rw.Write(b)
}

// Live reports whether the system is live.
func (s *Server) Live(rw http.ResponseWriter, req *http.Request) {
        fmt.Fprintf(rw, "ok")
}

// Ready reports whether the server is ready.
func (s *Server) Ready(rw http.ResponseWriter, req *http.Request) {
        fmt.Fprintf(rw, "ok")
}

func getClientIata(req *http.Request) string {
        iata := req.URL.Query().Get("iata")
        if iata != "" && len(iata) == 3 && isValidName(iata) {
                return strings.ToLower(iata)
        }
        return ""
}

func isValidName(s string) bool {
        if s == "" {
                return false
        }
        if len(s) > 10 {
                return false
        }
        return validName.MatchString(s)
}

func isValidType(s string) bool {
        switch s {
        case "physical", "virtual":
                return true
        default:
                return false
        }
}

func isValidUplink(s string) bool {
        // Minimally make sure the uplink speed specification looks like some
        // numbers followed by "g".
        matched, _ := regexp.MatchString("[0-9]+g", s)
        return matched
}

func (s *Server) getCountry(req *http.Request) (string, error) {
        c := req.URL.Query().Get("country")
        if c != "" {
                return c, nil
        }
        c = req.Header.Get("X-AppEngine-Country")
        if c != "" {
                return c, nil
        }
        record, err := s.Maxmind.City(net.ParseIP(getClientIP(req)))
        if err != nil {
                return "", err
        }
        return record.Country.IsoCode, nil
}

func rawLatLon(req *http.Request) (string, string, error) {
        lat := req.URL.Query().Get("lat")
        lon := req.URL.Query().Get("lon")
        if lat != "" && lon != "" {
                return lat, lon, nil
        }
        latlon := req.Header.Get("X-AppEngine-CityLatLong")
        if latlon != "0.000000,0.000000" {
                fields := strings.Split(latlon, ",")
                if len(fields) == 2 {
                        return fields[0], fields[1], nil
                }
        }
        return "", "", errLocationNotFound
}

func (s *Server) getLocation(req *http.Request) (float64, float64, error) {
        rlat, rlon, err := rawLatLon(req)
        if err == nil {
                lat, errLat := strconv.ParseFloat(rlat, 64)
                lon, errLon := strconv.ParseFloat(rlon, 64)
                if errLat != nil || errLon != nil {
                        return 0, 0, errLocationFormat
                }
                return lat, lon, nil
        }
        // Fall back to lookup with request IP.
        record, err := s.Maxmind.City(net.ParseIP(getClientIP(req)))
        if err != nil {
                return 0, 0, err
        }
        return record.Location.Latitude, record.Location.Longitude, nil
}

func writeResponse(rw http.ResponseWriter, resp interface{}) {
        b, err := json.MarshalIndent(resp, "", "  ")
        // NOTE: marshal can only fail on incompatible types, like functions. The
        // panic will be caught by the http server handler.
        rtx.PanicOnError(err, "failed to marshal response")
        rw.Write(b)
}

func checkIP(ip string) string {
        if net.ParseIP(ip) != nil {
                return ip
        }
        return ""
}

func getClientIP(req *http.Request) string {
        // Use given IP parameter.
        rawip := req.URL.Query().Get("ipv4")
        if rawip != "" {
                return rawip
        }
        // Use AppEngine's forwarded client address.
        fwdIPs := strings.Split(req.Header.Get("X-Forwarded-For"), ", ")
        if fwdIPs[0] != "" {
                return fwdIPs[0]
        }
        // Use remote client address.
        hip, _, _ := net.SplitHostPort(req.RemoteAddr)
        return hip
}

func getProbability(req *http.Request) float64 {
        prob := req.URL.Query().Get("probability")
        if prob == "" {
                return 1.0
        }
        p, err := strconv.ParseFloat(prob, 64)
        if err != nil {
                return 1.0
        }
        return p
}

func getPorts(req *http.Request) []string {
        result := []string{}
        ports := req.URL.Query()["ports"]
        for _, port := range ports {
                // Verify this is a valid number.
                _, err := strconv.ParseInt(port, 10, 64)
                if err != nil {
                        // Skip if not.
                        continue
                }
                result = append(result, port)
        }
        if len(result) == 0 {
                return []string{"9990"} // default port
        }
        return result
}

1	package handler
2
3	import (
4	"context"
5	"encoding/json"
6	"errors"
7	"fmt"
8	"log"
9	"net"
10	"net/http"
11	"regexp"
12	"strconv"
13	"strings"
14
15	"github.com/Masterminds/semver/v3"
16	v0 "github.com/m-lab/autojoin/api/v0"
17	"github.com/m-lab/autojoin/iata"
18	"github.com/m-lab/autojoin/internal/adminx"
19	"github.com/m-lab/autojoin/internal/dnsname"
20	"github.com/m-lab/autojoin/internal/dnsx"
21	"github.com/m-lab/autojoin/internal/dnsx/dnsiface"
22	"github.com/m-lab/autojoin/internal/register"
23	"github.com/m-lab/gcp-service-discovery/discovery"
24	"github.com/m-lab/go/host"
25	"github.com/m-lab/go/rtx"
26	v2 "github.com/m-lab/locate/api/v2"
27	"github.com/m-lab/uuid-annotator/annotator"
28	"github.com/oschwald/geoip2-golang"
29	)
30
31	var (
32	errLocationNotFound = errors.New("location not found")
33	errLocationFormat = errors.New("location could not be parsed")
34
35	validName = regexp.MustCompile(`[a-zA-Z0-9]+`)
36	)
37
38	// Server maintains shared state for the server.
39	type Server struct {
40	Project string
41	Iata IataFinder
42	Maxmind MaxmindFinder
43	ASN ASNFinder
44	DNS dnsiface.Service
45	minVersion *semver.Version
46
47	sm ServiceAccountSecretManager
48	dnsTracker DNSTracker
49	dsm Datastore
50	}
51
52	// ASNFinder is an interface used by the Server to manage ASN information.
53	type ASNFinder interface {
54	AnnotateIP(src string) *annotator.Network
55	Reload(ctx context.Context)
56	}
57
58	// MaxmindFinder is an interface used by the Server to manage Maxmind information.
59	type MaxmindFinder interface {
60	City(ip net.IP) (*geoip2.City, error)
61	Reload(ctx context.Context) error
62	}
63
64	// IataFinder is an interface used by the Server to manage IATA information.
65	type IataFinder interface {
66	Lookup(country string, lat, lon float64) (string, error)
67	Find(iata string) (iata.Row, error)
68	Load(ctx context.Context) error
69	}
70
71	type DNSTracker interface {
72	Update(string, []string) error
73	Delete(string) error
74	List() ([]string, [][]string, error)
75	}
76
77	// ServiceAccountSecretManager is an interface used by the server to allocate service account keys.
78	type ServiceAccountSecretManager interface {
79	LoadOrCreateKey(ctx context.Context, org string) (string, error)
80	}
81
82	type Datastore interface {
83	GetOrganization(ctx context.Context, name string) (*adminx.Organization, error)
84	}
85
86	// NewServer creates a new Server instance for request handling.
87	func NewServer(project string, finder IataFinder, maxmind MaxmindFinder, asn ASNFinder,
88	ds dnsiface.Service, tracker DNSTracker, sm ServiceAccountSecretManager, dsm Datastore,
89	minVersion string) *Server {
90	v, err := semver.NewVersion(minVersion)
91	rtx.Must(err, "invalid minimum version")	1✔
92	return &Server{	1✔
93	Project: project,	1✔
94	Iata: finder,	1✔
95	Maxmind: maxmind,	1✔
96	ASN: asn,	1✔
97	DNS: ds,	1✔
98	sm: sm,	1✔
99	minVersion: v,	1✔
100		1✔
101	dnsTracker: tracker,	1✔
102	dsm: dsm,	1✔
103	}	1✔
104	}	1✔
105		1✔
106	// Reload reloads all resources used by the Server.	1✔
107	func (s *Server) Reload(ctx context.Context) {
108	s.Iata.Load(ctx)
109	s.Maxmind.Reload(ctx)	1✔
110	}	1✔
111		1✔
112	// Lookup is a handler used to find the nearest IATA given client IP or lat/lon metadata.	1✔
113	func (s Server) Lookup(rw http.ResponseWriter, req http.Request) {
114	resp := v0.LookupResponse{}
115	country, err := s.getCountry(req)	1✔
116	if country == "" \|\| err != nil {	1✔
117	resp.Error = &v2.Error{	1✔
118	Type: "?country=<country>",	2✔
119	Title: "could not determine country from request",	1✔
120	Status: http.StatusBadRequest,	1✔
121	}	1✔
122	rw.WriteHeader(resp.Error.Status)	1✔
123	writeResponse(rw, resp)	1✔
124	return	1✔
125	}	1✔
126	lat, lon, err := s.getLocation(req)	1✔
127	if err != nil {	1✔
128	resp.Error = &v2.Error{	1✔
129	Type: "?lat=<lat>&lon=<lon>",	2✔
130	Title: "could not determine lat/lon from request",	1✔
131	Status: http.StatusBadRequest,	1✔
132	}	1✔
133	rw.WriteHeader(resp.Error.Status)	1✔
134	writeResponse(rw, resp)	1✔
135	return	1✔
136	}	1✔
137	code, err := s.Iata.Lookup(country, lat, lon)	1✔
138	if err != nil {	1✔
139	resp.Error = &v2.Error{	1✔
140	Type: "internal error",	2✔
141	Title: "could not determine iata from request",	1✔
142	Status: http.StatusInternalServerError,	1✔
143	}	1✔
144	rw.WriteHeader(resp.Error.Status)	1✔
145	writeResponse(rw, resp)	1✔
146	return	1✔
147	}	1✔
148	resp.Lookup = &v0.Lookup{	1✔
149	IATA: code,	1✔
150	}	1✔
151	writeResponse(rw, resp)	1✔
152	}	1✔
153		1✔
154	// Register handler is used by autonodes to register their hostname with M-Lab
155	// on startup and receive additional needed configuration metadata.
156	func (s Server) Register(rw http.ResponseWriter, req http.Request) {
157	// All replies, errors and successes, should be json.
158	rw.Header().Set("Content-Type", "application/json")	1✔
159		1✔
160	resp := v0.RegisterResponse{}	1✔
161		1✔
162	// Check version first.	1✔
163	versionStr := req.URL.Query().Get("version")	1✔
164	// If no version is provided, default to v0.0.0. This allows existing clients	1✔
165	// that do not provide the version yet to keep working until a minVersion is set.	1✔
166	if versionStr == "" {	1✔
167	versionStr = "v0.0.0"	1✔
168	}	2✔
169		1✔
170	// Parse the provided version.	1✔
171	clientVersion, err := semver.NewVersion(versionStr)
172	if err != nil {
173	resp.Error = &v2.Error{	1✔
174	Type: "version.invalid",	2✔
175	Title: "invalid version format - must be semantic version (e.g. v1.2.3)",	1✔
176	Detail: err.Error(),	1✔
177	Status: http.StatusBadRequest,	1✔
178	}	1✔
179	rw.WriteHeader(resp.Error.Status)	1✔
180	writeResponse(rw, resp)	1✔
181	return	1✔
182	}	1✔
183		1✔
184	if clientVersion.LessThan(s.minVersion) {	1✔
185	resp.Error = &v2.Error{
186	Type: "version.outdated",	2✔
187	Title: fmt.Sprintf("version %s is below minimum required version %s",	1✔
188	clientVersion.String(), s.minVersion.String()),	1✔
189	Status: http.StatusForbidden,	1✔
190	}	1✔
191	rw.WriteHeader(resp.Error.Status)	1✔
192	writeResponse(rw, resp)	1✔
193	return	1✔
194	}	1✔
195		1✔
196	param := &register.Params{Project: s.Project}	1✔
197	param.Service = req.URL.Query().Get("service")
198	if !isValidName(param.Service) {	1✔
199	resp.Error = &v2.Error{	1✔
200	Type: "?service=<service>",	2✔
201	Title: "could not determine service from request",	1✔
202	Status: http.StatusBadRequest,	1✔
203	}	1✔
204	rw.WriteHeader(resp.Error.Status)	1✔
205	writeResponse(rw, resp)	1✔
206	return	1✔
207	}	1✔
208		1✔
209	// Get the organization from the context.	1✔
210	org, ok := req.Context().Value(orgContextKey).(string)
211	if !ok {
212	resp.Error = &v2.Error{	1✔
213	Type: "auth.context",	1✔
214	Title: "missing organization in context",	×
215	Status: http.StatusInternalServerError,	×
216	}	×
217	rw.WriteHeader(resp.Error.Status)	×
218	writeResponse(rw, resp)	×
219	return	×
220	}	×
UNCOV 221	param.Org = org	×
UNCOV 222	param.IPv6 = checkIP(req.URL.Query().Get("ipv6")) // optional.	×
223	param.IPv4 = checkIP(getClientIP(req))	1✔
224	ip := net.ParseIP(param.IPv4)	1✔
225	if ip == nil \|\| ip.To4() == nil {	1✔
226	resp.Error = &v2.Error{	1✔
227	Type: "?ipv4=<ipv4>",	2✔
228	Title: "could not determine client ipv4 from request",	1✔
229	Status: http.StatusBadRequest,	1✔
230	}	1✔
231	rw.WriteHeader(resp.Error.Status)	1✔
232	writeResponse(rw, resp)	1✔
233	return	1✔
234	}	1✔
235	param.Type = req.URL.Query().Get("type")	1✔
236	if !isValidType(param.Type) {	1✔
237	resp.Error = &v2.Error{	1✔
238	Type: "?type=<type>",	2✔
239	Title: "invalid machine type from request",	1✔
240	Status: http.StatusBadRequest,	1✔
241	}	1✔
242	rw.WriteHeader(resp.Error.Status)	1✔
243	writeResponse(rw, resp)	1✔
244	return	1✔
245	}	1✔
246	param.Uplink = req.URL.Query().Get("uplink")	1✔
247	if !isValidUplink(param.Uplink) {	1✔
248	resp.Error = &v2.Error{	1✔
249	Type: "?uplink=<uplink>",	2✔
250	Title: "invalid uplink speed from request",	1✔
251	Status: http.StatusBadRequest,	1✔
252	}	1✔
253	rw.WriteHeader(resp.Error.Status)	1✔
254	writeResponse(rw, resp)	1✔
255	return	1✔
256	}	1✔
257	iata := getClientIata(req)	1✔
258	if iata == "" {	1✔
259	resp.Error = &v2.Error{	1✔
260	Type: "?iata=<iata>",	1✔
261	Title: "could not determine iata from request",	×
262	Status: http.StatusBadRequest,	×
263	}	×
264	rw.WriteHeader(resp.Error.Status)	×
265	writeResponse(rw, resp)	×
266	return	×
267	}	×
UNCOV 268	row, err := s.Iata.Find(iata)	×
UNCOV 269	if err != nil {	×
270	resp.Error = &v2.Error{	1✔
271	Type: "iata.find",	2✔
272	Title: "could not find given iata in dataset",	1✔
273	Status: http.StatusInternalServerError,	1✔
274	}	1✔
275	rw.WriteHeader(resp.Error.Status)	1✔
276	writeResponse(rw, resp)	1✔
277	return	1✔
278	}	1✔
279	param.Metro = row	1✔
280	record, err := s.Maxmind.City(ip)	1✔
281	if err != nil {	1✔
282	resp.Error = &v2.Error{	1✔
283	Type: "maxmind.city",	2✔
284	Title: "could not find city metadata from ip",	1✔
285	Status: http.StatusInternalServerError,	1✔
286	}	1✔
287	rw.WriteHeader(resp.Error.Status)	1✔
288	writeResponse(rw, resp)	1✔
289	return	1✔
290	}	1✔
291	param.Geo = record	1✔
292	param.Network = s.ASN.AnnotateIP(param.IPv4)	1✔
293		1✔
294	// Get the organization probability multiplier.	1✔
295	orgEntity, err := s.dsm.GetOrganization(req.Context(), param.Org)	1✔
296	orgMultiplier := 1.0	1✔
297	if err == nil && orgEntity != nil && orgEntity.ProbabilityMultiplier != nil {	1✔
298	orgMultiplier = *orgEntity.ProbabilityMultiplier	1✔
299	}	2✔
300	// Assign the probability by multiplying the org multiplier with the	1✔
301	// probability requested by the client.	1✔
302	param.Probability = getProbability(req) * orgMultiplier
303	r := register.CreateRegisterResponse(param)
304		1✔
305	key, err := s.sm.LoadOrCreateKey(req.Context(), param.Org)	1✔
306	if err != nil {	1✔
307	resp.Error = &v2.Error{	1✔
308	Type: "load.serviceaccount.key",	2✔
309	Title: "could not load service account key for node",	1✔
310	Status: http.StatusInternalServerError,	1✔
311	}	1✔
312	log.Println("loading service account key failure:", err)	1✔
313	rw.WriteHeader(resp.Error.Status)	1✔
314	writeResponse(rw, resp)	1✔
315	return	1✔
316	}	1✔
317	r.Registration.Credentials = &v0.Credentials{	1✔
318	ServiceAccountKey: key,	1✔
319	}	1✔
320		1✔
321	// Register the hostname under the organization zone.	1✔
322	m := dnsx.NewManager(s.DNS, s.Project, dnsname.OrgZone(param.Org, s.Project))	1✔
323	_, err = m.Register(req.Context(), r.Registration.Hostname+".", param.IPv4, param.IPv6)	1✔
324	if err != nil {	1✔
325	resp.Error = &v2.Error{	1✔
326	Type: "dns.register",	2✔
327	Title: "could not register dynamic hostname",	1✔
328	Status: http.StatusInternalServerError,	1✔
329	}	1✔
330	log.Println("dns register failure:", err)	1✔
331	rw.WriteHeader(resp.Error.Status)	1✔
332	writeResponse(rw, resp)	1✔
333	return	1✔
334	}	1✔
335		1✔
336	// Add the hostname to the DNS tracker.	1✔
337	err = s.dnsTracker.Update(r.Registration.Hostname, getPorts(req))
338	if err != nil {
339	resp.Error = &v2.Error{	1✔
340	Type: "tracker.gc",	2✔
341	Title: "could not update DNS tracker",	1✔
342	Status: http.StatusInternalServerError,	1✔
343	}	1✔
344	log.Println("dns gc update failure:", err)	1✔
345	rw.WriteHeader(resp.Error.Status)	1✔
346	writeResponse(rw, resp)	1✔
347	return	1✔
348	}	1✔
349		1✔
350	b, _ := json.MarshalIndent(r, "", " ")	1✔
351	rw.Write(b)
352	}	1✔
353		1✔
354	// Delete handler is used by operators to delete a previously registered
355	// hostname from DNS.
356	func (s Server) Delete(rw http.ResponseWriter, req http.Request) {
357	// All replies, errors and successes, should be json.
358	rw.Header().Set("Content-Type", "application/json")	1✔
359		1✔
360	resp := v0.DeleteResponse{}	1✔
361	hostname := req.URL.Query().Get("hostname")	1✔
362	name, err := host.Parse(hostname)	1✔
363	if err != nil {	1✔
364	resp.Error = &v2.Error{	1✔
365	Type: "dns.delete",	2✔
366	Title: "failed to parse hostname",	1✔
367	Detail: err.Error(),	1✔
368	Status: http.StatusBadRequest,	1✔
369	}	1✔
370	log.Println("dns delete (parse) failure:", err)	1✔
371	rw.WriteHeader(resp.Error.Status)	1✔
372	writeResponse(rw, resp)	1✔
373	return	1✔
374	}	1✔
375		1✔
376	m := dnsx.NewManager(s.DNS, s.Project, dnsname.OrgZone(name.Org, s.Project))	1✔
377	_, err = m.Delete(req.Context(), name.StringAll()+".")
378	if err != nil {	1✔
379	resp.Error = &v2.Error{	1✔
380	Type: "dns.delete",	2✔
381	Title: "failed to delete hostname",	1✔
382	Detail: err.Error(),	1✔
383	Status: http.StatusInternalServerError,	1✔
384	}	1✔
385	log.Println("dns delete failure:", err)	1✔
386	rw.WriteHeader(resp.Error.Status)	1✔
387	writeResponse(rw, resp)	1✔
388	return	1✔
389	}	1✔
390		1✔
391	err = s.dnsTracker.Delete(name.StringAll())	1✔
392	if err != nil {
393	resp.Error = &v2.Error{	1✔
394	Type: "tracker.gc",	2✔
395	Title: "failed to delete hostname from DNS tracker",	1✔
396	Detail: err.Error(),	1✔
397	Status: http.StatusInternalServerError,	1✔
398	}	1✔
399	log.Println("dns gc delete failure:", err)	1✔
400	rw.WriteHeader(resp.Error.Status)	1✔
401	writeResponse(rw, resp)	1✔
402	return	1✔
403	}	1✔
404		1✔
405	b, err := json.MarshalIndent(resp, "", " ")	1✔
406	rtx.Must(err, "failed to marshal DNS delete response")
407	rw.Write(b)	1✔
408	}	1✔
409		1✔
410	// List handler is used by monitoring to generate a list of known, active
411	// hostnames previously registered with the Autojoin API.
412	func (s Server) List(rw http.ResponseWriter, req http.Request) {
413	// Set CORS policy to allow third-party websites to use returned resources.
414	rw.Header().Set("Content-Type", "application/json")	1✔
415	rw.Header().Set("Access-Control-Allow-Origin", "*")	1✔
416	rw.Header().Set("Cache-Control", "no-store") // Prevent caching of result.	1✔
417		1✔
418	configs := []discovery.StaticConfig{}	1✔
419	resp := v0.ListResponse{}	1✔
420	hosts, ports, err := s.dnsTracker.List()	1✔
421	if err != nil {	1✔
422	resp.Error = &v2.Error{	1✔
423	Type: "list",	2✔
424	Title: "failed to list node records",	1✔
425	Detail: err.Error(),	1✔
426	Status: http.StatusInternalServerError,	1✔
427	}	1✔
428	log.Println("list failure:", err)	1✔
429	rw.WriteHeader(resp.Error.Status)	1✔
430	writeResponse(rw, resp)	1✔
431	return	1✔
432	}	1✔
433		1✔
434	org := req.URL.Query().Get("org")	1✔
435	format := req.URL.Query().Get("format")
436	sites := map[string]bool{}	1✔
437		1✔
438	// Create a prometheus StaticConfig for each known host.	1✔
439	for i := range hosts {	1✔
440	h, err := host.Parse(hosts[i])	1✔
441	if err != nil {	2✔
442	continue	1✔
443	}	2✔
444	if org != "" && org != h.Org {	1✔
445	// Skip hosts that are not part of the given org.
446	continue	2✔
447	}	1✔
448	sites[h.Site] = true	1✔
449	if format == "script-exporter" {
450	// NOTE: do not assign any ports for script exporter.	1✔
451	ports[i] = []string{""}	2✔
452	} else {	1✔
453	// Convert port strings to ":<port>".	1✔
454	p := []string{}	2✔
455	for j := range ports[i] {	1✔
456	p = append(p, ":"+ports[i][j])	1✔
457	}	2✔
458	ports[i] = p	1✔
459	}	1✔
460	for _, port := range ports[i] {	1✔
461	labels := map[string]string{
462	"machine": hosts[i],	2✔
463	"type": "virtual",	1✔
464	"deployment": "byos",	1✔
465	"managed": "none",	1✔
466	"org": h.Org,	1✔
467	}	1✔
468	if req.URL.Query().Get("service") != "" {	1✔
469	labels["service"] = req.URL.Query().Get("service")	1✔
470	}	2✔
471	// We create one record per host to add a unique "machine" label to each one.	1✔
472	configs = append(configs, discovery.StaticConfig{	1✔
473	Targets: []string{hosts[i] + port},
474	Labels: labels,	1✔
475	})	1✔
476	}	1✔
477	}	1✔
478
479	var results interface{}
480	switch format {
481	case "script-exporter":	1✔
482	fallthrough	1✔
483	case "blackbox":	1✔
484	fallthrough	1✔
485	case "prometheus":	1✔
486	results = configs	1✔
487	case "servers":	1✔
488	resp.Servers = hosts	1✔
489	results = resp	1✔
490	case "sites":	1✔
491	for k := range sites {	1✔
492	resp.Sites = append(resp.Sites, k)	1✔
493	}	2✔
494	results = resp	1✔
495	default:	1✔
496	resp.Servers = hosts	1✔
497	results = resp	1✔
498	}	1✔
499	// Generate as JSON; the list may be empty.	1✔
500	b, err := json.MarshalIndent(results, "", " ")
501	rtx.Must(err, "failed to marshal DNS delete response")
502	rw.Write(b)	1✔
503	}	1✔
504		1✔
505	// Live reports whether the system is live.
506	func (s Server) Live(rw http.ResponseWriter, req http.Request) {
507	fmt.Fprintf(rw, "ok")
508	}	1✔
509		1✔
510	// Ready reports whether the server is ready.	1✔
511	func (s Server) Ready(rw http.ResponseWriter, req http.Request) {
512	fmt.Fprintf(rw, "ok")
513	}	1✔
514		1✔
515	func getClientIata(req *http.Request) string {	1✔
516	iata := req.URL.Query().Get("iata")
517	if iata != "" && len(iata) == 3 && isValidName(iata) {	1✔
518	return strings.ToLower(iata)	1✔
519	}	2✔
520	return ""	1✔
521	}	1✔
UNCOV 522		×
523	func isValidName(s string) bool {
524	if s == "" {
525	return false	1✔
526	}	2✔
527	if len(s) > 10 {	1✔
528	return false	1✔
529	}	2✔
530	return validName.MatchString(s)	1✔
531	}	1✔
532		1✔
533	func isValidType(s string) bool {
534	switch s {
535	case "physical", "virtual":	1✔
536	return true	1✔
537	default:	1✔
538	return false	1✔
539	}	1✔
540	}	1✔
541
542	func isValidUplink(s string) bool {
543	// Minimally make sure the uplink speed specification looks like some
544	// numbers followed by "g".	1✔
545	matched, _ := regexp.MatchString("[0-9]+g", s)	1✔
546	return matched	1✔
547	}	1✔
548		1✔
549	func (s Server) getCountry(req http.Request) (string, error) {
550	c := req.URL.Query().Get("country")	1✔
551	if c != "" {	1✔
552	return c, nil	2✔
553	}	1✔
554	c = req.Header.Get("X-AppEngine-Country")	1✔
555	if c != "" {	1✔
556	return c, nil	2✔
557	}	1✔
558	record, err := s.Maxmind.City(net.ParseIP(getClientIP(req)))	1✔
559	if err != nil {	1✔
560	return "", err	2✔
561	}	1✔
562	return record.Country.IsoCode, nil	1✔
563	}	1✔
564
565	func rawLatLon(req *http.Request) (string, string, error) {
566	lat := req.URL.Query().Get("lat")	1✔
567	lon := req.URL.Query().Get("lon")	1✔
568	if lat != "" && lon != "" {	1✔
569	return lat, lon, nil	2✔
570	}	1✔
571	latlon := req.Header.Get("X-AppEngine-CityLatLong")	1✔
572	if latlon != "0.000000,0.000000" {	1✔
573	fields := strings.Split(latlon, ",")	2✔
574	if len(fields) == 2 {	1✔
575	return fields[0], fields[1], nil	2✔
576	}	1✔
577	}	1✔
578	return "", "", errLocationNotFound
579	}	1✔
580
581	func (s Server) getLocation(req http.Request) (float64, float64, error) {
582	rlat, rlon, err := rawLatLon(req)	1✔
583	if err == nil {	1✔
584	lat, errLat := strconv.ParseFloat(rlat, 64)	2✔
585	lon, errLon := strconv.ParseFloat(rlon, 64)	1✔
586	if errLat != nil \|\| errLon != nil {	1✔
587	return 0, 0, errLocationFormat	2✔
588	}	1✔
589	return lat, lon, nil	1✔
590	}	1✔
591	// Fall back to lookup with request IP.
592	record, err := s.Maxmind.City(net.ParseIP(getClientIP(req)))
593	if err != nil {	1✔
594	return 0, 0, err	2✔
595	}	1✔
596	return record.Location.Latitude, record.Location.Longitude, nil	1✔
597	}	1✔
598
599	func writeResponse(rw http.ResponseWriter, resp interface{}) {
600	b, err := json.MarshalIndent(resp, "", " ")	1✔
601	// NOTE: marshal can only fail on incompatible types, like functions. The	1✔
602	// panic will be caught by the http server handler.	1✔
603	rtx.PanicOnError(err, "failed to marshal response")	1✔
604	rw.Write(b)	1✔
605	}	1✔
606		1✔
607	func checkIP(ip string) string {
608	if net.ParseIP(ip) != nil {	1✔
609	return ip	2✔
610	}	1✔
611	return ""	1✔
612	}	1✔
613
614	func getClientIP(req *http.Request) string {
615	// Use given IP parameter.	1✔
616	rawip := req.URL.Query().Get("ipv4")	1✔
617	if rawip != "" {	1✔
618	return rawip	2✔
619	}	1✔
620	// Use AppEngine's forwarded client address.	1✔
621	fwdIPs := strings.Split(req.Header.Get("X-Forwarded-For"), ", ")
622	if fwdIPs[0] != "" {	1✔
623	return fwdIPs[0]	2✔
624	}	1✔
625	// Use remote client address.	1✔
626	hip, _, _ := net.SplitHostPort(req.RemoteAddr)
627	return hip	1✔
628	}	1✔
629
630	func getProbability(req *http.Request) float64 {
631	prob := req.URL.Query().Get("probability")	1✔
632	if prob == "" {	1✔
633	return 1.0	2✔
634	}	1✔
635	p, err := strconv.ParseFloat(prob, 64)	1✔
636	if err != nil {	1✔
637	return 1.0	2✔
638	}	1✔
639	return p	1✔
640	}	1✔
641
642	func getPorts(req *http.Request) []string {
643	result := []string{}	1✔
644	ports := req.URL.Query()["ports"]	1✔
645	for _, port := range ports {	1✔
646	// Verify this is a valid number.	2✔
647	_, err := strconv.ParseInt(port, 10, 64)	1✔
648	if err != nil {	1✔
649	// Skip if not.	2✔
650	continue	1✔
651	}	1✔
652	result = append(result, port)
653	}
654	if len(result) == 0 {	2✔
655	return []string{"9990"} // default port	1✔
656	}	1✔
657	return result
658	}	1✔

m-lab / autojoin / 16273924685

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous