monitoring: Add OpenMetrics exporter

This patch makes chasquid's monitoring server expose an OpenMetrics
metrics endpoint.

It adds a new package "expvarom" which implements an HTTP handler that
exports expvar variables in the OpenMetrics text format.

Then, the handler is registered by the monitoring server at /metrics
(where most things expect it to be).

The existing exported variables are also extended with descriptions,
which is optional, but improves the readability of the metrics.

internal/expvarom/expvarom.go

@@ -0,0 +1,224 @@
+// Package expvarom implements an OpenMetrics HTTP exporter for the variables
+// from the expvar package.
+//
+// This is useful for small servers that want to support both packages with
+// simple enough variables, without introducing any dependencies beyond the
+// standard library.
+//
+// Some functions to add descriptions and map labels are exported for
+// convenience, but their usage is optional.
+//
+// For more complex usage (like histograms, counters vs. gauges, etc.), use
+// the OpenMetrics libraries directly.
+//
+// The exporter uses the text-based format, as documented in:
+// https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format
+//
+// Note the adoption of that format as OpenMetrics' one isn't finalized yet,
+// and it is possible that it will change in the future.
+//
+// Backwards compatibility is NOT guaranteed, until the format is fully
+// standarized.
+package expvarom
+
+import (
+	"expvar"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"unicode/utf8"
+)
+
+type exportedVar struct {
+	Name      string
+	Desc      string
+	LabelName string
+
+	I *expvar.Int
+	F *expvar.Float
+	M *expvar.Map
+}
+
+var (
+	infoMu        = sync.Mutex{}
+	descriptions  = map[string]string{}
+	mapLabelNames = map[string]string{}
+)
+
+// MetricsHandler implements an http.HandlerFunc which serves the registered
+// metrics, using the OpenMetrics text-based format.
+func MetricsHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4")
+
+	vars := []exportedVar{}
+	ignored := []string{}
+	expvar.Do(func(kv expvar.KeyValue) {
+		evar := exportedVar{
+			Name: metricNameToOM(kv.Key),
+		}
+		switch value := kv.Value.(type) {
+		case *expvar.Int:
+			evar.I = value
+		case *expvar.Float:
+			evar.F = value
+		case *expvar.Map:
+			evar.M = value
+		default:
+			// Unsupported type, ignore this variable.
+			ignored = append(ignored, evar.Name)
+			return
+		}
+
+		infoMu.Lock()
+		evar.Desc = descriptions[kv.Key]
+		evar.LabelName = mapLabelNames[kv.Key]
+		infoMu.Unlock()
+
+		// OM maps need a label name, while expvar ones do not. If we weren't
+		// told what to use, use a generic "key".
+		if evar.LabelName == "" {
+			evar.LabelName = "key"
+		}
+
+		vars = append(vars, evar)
+	})
+
+	// Sort the variables for reproducibility and readability.
+	sort.Slice(vars, func(i, j int) bool {
+		return vars[i].Name < vars[j].Name
+	})
+
+	for _, v := range vars {
+		writeVar(w, &v)
+	}
+
+	fmt.Fprintf(w, "# Generated by expvarom\n")
+	fmt.Fprintf(w, "# EXPERIMENTAL - Format is not fully standard yet\n")
+	fmt.Fprintf(w, "# Ignored variables: %q\n", ignored)
+}
+
+func writeVar(w io.Writer, v *exportedVar) {
+	if v.Desc != "" {
+		fmt.Fprintf(w, "# HELP %s %s\n", v.Name, v.Desc)
+	}
+
+	if v.I != nil {
+		fmt.Fprintf(w, "%s %d\n\n", v.Name, v.I.Value())
+		return
+	}
+
+	if v.F != nil {
+		fmt.Fprintf(w, "%s %g\n\n", v.Name, v.F.Value())
+		return
+	}
+
+	if v.M != nil {
+		count := 0
+		v.M.Do(func(kv expvar.KeyValue) {
+			vs := ""
+			switch value := kv.Value.(type) {
+			case *expvar.Int:
+				vs = strconv.FormatInt(value.Value(), 10)
+			case *expvar.Float:
+				vs = strconv.FormatFloat(value.Value(), 'g', -1, 64)
+			default:
+				// We only support Int and Float in maps.
+				return
+			}
+
+			labelValue := quoteLabelValue(kv.Key)
+
+			fmt.Fprintf(w, "%s{%s=%s} %s\n",
+				v.Name, v.LabelName, labelValue, vs)
+			count++
+		})
+		if count > 0 {
+			fmt.Fprintf(w, "\n")
+		}
+	}
+}
+
+// metricNameToOM converts an expvar metric name into an OpenMetrics-compliant
+// metric name. The latter is more restrictive, as it must match the regexp
+// "[a-zA-Z_:][a-zA-Z0-9_:]*", AND the ':' is not allowed for a direct
+// exporter.
+//
+// https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
+func metricNameToOM(name string) string {
+	n := ""
+	for _, c := range name {
+		if (c >= 'a' && c <= 'z') ||
+			(c >= 'A' && c <= 'Z') ||
+			(c >= '0' && c <= '9') ||
+			c == '_' {
+			n += string(c)
+		} else {
+			n += "_"
+		}
+	}
+
+	// If it begins with a number, prepend 'i' as a compromise.
+	if len(n) > 0 && n[0] >= '0' && n[0] <= '9' {
+		n = "i" + n
+	}
+
+	return n
+}
+
+// According to the spec, we only need to replace these 3 characters in label
+// values.
+var labelValueReplacer = strings.NewReplacer(
+	`\`, `\\`,
+	`"`, `\"`,
+	"\n", `\n`)
+
+// quoteLabelValue takes an arbitrary string, and quotes it so it can be
+// used as a label value. Output includes the wrapping `"`.
+func quoteLabelValue(v string) string {
+	// The spec requires label values to be valid UTF8, with `\`, `"` and "\n"
+	// escaped.  If it's invalid UTF8, hard-quote it first.  This will result
+	// in uglier looking values, but they will be well formed.
+	if !utf8.ValidString(v) {
+		v = strconv.QuoteToASCII(v)
+		v = v[1 : len(v)-1]
+	}
+
+	return `"` + labelValueReplacer.Replace(v) + `"`
+}
+
+// NewInt registers a new expvar.Int variable, with the given description.
+func NewInt(name, desc string) *expvar.Int {
+	infoMu.Lock()
+	descriptions[name] = desc
+	infoMu.Unlock()
+	return expvar.NewInt(name)
+}
+
+// NewFloat registers a new expvar.Float variable, with the given description.
+func NewFloat(name, desc string) *expvar.Float {
+	infoMu.Lock()
+	descriptions[name] = desc
+	infoMu.Unlock()
+	return expvar.NewFloat(name)
+}
+
+// NewMap registers a new expvar.Map variable, with the given label
+// name and description.
+func NewMap(name, labelName, desc string) *expvar.Map {
+	// Prevent accidents when using the description as the label name.
+	if strings.Contains(labelName, " ") {
+		panic(fmt.Sprintf(
+			"label name has spaces, mix up with the description? %q",
+			labelName))
+	}
+
+	infoMu.Lock()
+	descriptions[name] = desc
+	mapLabelNames[name] = labelName
+	infoMu.Unlock()
+	return expvar.NewMap(name)
+}