Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion runtimes/go/appruntime/infrasdk/metrics/aws/cloudwatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,24 @@ func (x *Exporter) Export(ctx context.Context, collected []metrics.CollectedMetr
func (x *Exporter) getMetricData(now time.Time, collected []metrics.CollectedMetric) []types.MetricDatum {
data := make([]types.MetricDatum, 0, len(collected))

// svcLabels is set per-metric in the loop below and captured by doAdd.
var svcLabels map[string][]metrics.KeyValue

doAdd := func(val float64, metricName string, baseDims []types.Dimension, svcIdx uint16) {
svcName := x.svcs[svcIdx]
dims := make([]types.Dimension, len(baseDims)+1)
copy(dims, baseDims)
dims[len(baseDims)] = types.Dimension{
Name: aws.String("service"),
Value: aws.String(x.svcs[svcIdx]),
Value: aws.String(svcName),
}
if extra, ok := svcLabels[svcName]; ok {
for _, kv := range extra {
dims = append(dims, types.Dimension{
Name: aws.String(kv.Key),
Value: aws.String(kv.Value),
})
}
}
data = append(data, types.MetricDatum{
MetricName: aws.String(metricName),
Expand All @@ -97,6 +109,7 @@ func (x *Exporter) getMetricData(now time.Time, collected []metrics.CollectedMet
}

for _, m := range collected {
svcLabels = m.ServiceLabels
dims := make([]types.Dimension, len(x.containerMetadataDims), len(x.containerMetadataDims)+len(m.Labels))
copy(dims, x.containerMetadataDims)
for _, label := range m.Labels {
Expand Down
8 changes: 7 additions & 1 deletion runtimes/go/appruntime/infrasdk/metrics/datadog/datadog.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,15 @@ func (x *Exporter) getMetricData(now time.Time, collected []metrics.CollectedMet
}

doAdd := func(val float64, metricName string, baseLabels []string, svcIdx uint16) {
svcName := x.svcs[svcIdx]
labels := make([]string, len(baseLabels)+1)
copy(labels, baseLabels)
labels[len(baseLabels)] = "service:" + x.svcs[svcIdx]
labels[len(baseLabels)] = "service:" + svcName
if extra, ok := m.ServiceLabels[svcName]; ok {
for _, kv := range extra {
labels = append(labels, kv.Key+":"+kv.Value)
}
}
if m.Info.Type() == metrics.CounterType {
key := tsSvcKey{tsID: m.TimeSeriesID, svc: svcIdx}
lastVal := x.lastValue[key]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,11 +143,17 @@ func (x *Exporter) getMetricData(newCounterStart, endTime time.Time, collected [
metricType = "custom.googleapis.com/" + cloudMetricName

doAdd := func(val *monitoringpb.TypedValue, svcIdx uint16) {
svcName := x.svcs[svcIdx]
labels := make(map[string]string, len(baseLabels)+1)
for k, v := range baseLabels {
labels[k] = v
}
labels["service"] = x.svcs[svcIdx]
labels["service"] = svcName
if extra, ok := m.ServiceLabels[svcName]; ok {
for _, kv := range extra {
labels[kv.Key] = kv.Value
}
}

data = append(data, &monitoringpb.TimeSeries{
MetricKind: kind,
Expand Down
11 changes: 11 additions & 0 deletions runtimes/go/appruntime/infrasdk/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,17 @@ func (mgr *Manager) collectNow(ctx context.Context) {
}

m := mgr.reg.Collect()

// Attach service labels to collected metrics so exporters can enrich
// per-service time series with additional labels (e.g., team, cost_center).
// The same snapshot is shared across all metrics — this is safe because
// exporters only read from it and collection+export is sequential.
if svcLabels := mgr.reg.ServiceLabels(); svcLabels != nil {
for i := range m {
m[i].ServiceLabels = svcLabels
}
}

if err := mgr.exp.Export(ctx, m); err != nil {
mgr.rootLogger.Error().Err(err).Msg("unable to emit metrics")
} else {
Expand Down
12 changes: 11 additions & 1 deletion runtimes/go/appruntime/infrasdk/metrics/prometheus/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,20 @@ func (x *Exporter) Export(ctx context.Context, collected []metrics.CollectedMetr
func (x *Exporter) getMetricData(now time.Time, collected []metrics.CollectedMetric) []*prompb.TimeSeries {
data := make([]*prompb.TimeSeries, 0, len(collected))

// svcLabels is set per-metric in the loop below and captured by doAdd.
var svcLabels map[string][]metrics.KeyValue

doAdd := func(val float64, metricName string, baseLabels []*prompb.Label, svcIdx uint16) {
svcName := x.svcs[svcIdx]
labels := make([]*prompb.Label, len(baseLabels)+2)
copy(labels, baseLabels)
labels[len(baseLabels)] = &prompb.Label{Name: "__name__", Value: metricName}
labels[len(baseLabels)+1] = &prompb.Label{Name: "service", Value: x.svcs[svcIdx]}
labels[len(baseLabels)+1] = &prompb.Label{Name: "service", Value: svcName}
if extra, ok := svcLabels[svcName]; ok {
for _, kv := range extra {
labels = append(labels, &prompb.Label{Name: kv.Key, Value: kv.Value})
}
}
// Sort labels lexicographically by name, as required by some Prometheus implementations.
slices.SortFunc(labels, func(a, b *prompb.Label) int {
return strings.Compare(a.Name, b.Name)
Expand All @@ -96,6 +105,7 @@ func (x *Exporter) getMetricData(now time.Time, collected []metrics.CollectedMet
}

for _, m := range collected {
svcLabels = m.ServiceLabels
labels := make([]*prompb.Label, len(x.containerMetadataLabels))
copy(labels, x.containerMetadataLabels)
for _, label := range m.Labels {
Expand Down
106 changes: 106 additions & 0 deletions runtimes/go/appruntime/infrasdk/metrics/prometheus/prometheus_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,3 +193,109 @@ func TestGetMetricData(t *testing.T) {
})
}
}

func TestGetMetricData_ServiceLabels(t *testing.T) {
now := time.Now()
svcs := []string{"svc_a", "svc_b"}

svcLabels := map[string][]metrics.KeyValue{
"svc_a": {{Key: "team", Value: "backend"}},
}

t.Run("service labels included in output", func(t *testing.T) {
m := metrics.CollectedMetric{
Info: metricInfo{"test_counter", metrics.CounterType, 1},
Val: []int64{5},
ServiceLabels: svcLabels,
Valid: func() []atomic.Bool {
v := make([]atomic.Bool, 1)
v[0].Store(true)
return v
}(),
}

x := &Exporter{svcs: svcs}
got := x.getMetricData(now, []metrics.CollectedMetric{m})

if len(got) != 1 {
t.Fatalf("got %d items, want 1", len(got))
}

want := []*prompb.Label{
{Name: "__name__", Value: "test_counter"},
{Name: "service", Value: "svc_a"},
{Name: "team", Value: "backend"},
}
if !reflect.DeepEqual(got[0].Labels, want) {
t.Errorf("got labels %+v, want %+v", got[0].Labels, want)
}
})

t.Run("no service labels for unregistered service", func(t *testing.T) {
m := metrics.CollectedMetric{
Info: metricInfo{"test_counter", metrics.CounterType, 2},
Val: []int64{5},
ServiceLabels: svcLabels,
Valid: func() []atomic.Bool {
v := make([]atomic.Bool, 1)
v[0].Store(true)
return v
}(),
}

x := &Exporter{svcs: svcs}
got := x.getMetricData(now, []metrics.CollectedMetric{m})

if len(got) != 1 {
t.Fatalf("got %d items, want 1", len(got))
}

want := []*prompb.Label{
{Name: "__name__", Value: "test_counter"},
{Name: "service", Value: "svc_b"},
}
if !reflect.DeepEqual(got[0].Labels, want) {
t.Errorf("got labels %+v, want %+v", got[0].Labels, want)
}
})

t.Run("multi-service metric with service labels", func(t *testing.T) {
m := metrics.CollectedMetric{
Info: metricInfo{"test_counter", metrics.CounterType, 0},
Val: []int64{1, 2},
ServiceLabels: svcLabels,
Valid: func() []atomic.Bool {
v := make([]atomic.Bool, 2)
v[0].Store(true)
v[1].Store(true)
return v
}(),
}

x := &Exporter{svcs: svcs}
got := x.getMetricData(now, []metrics.CollectedMetric{m})

if len(got) != 2 {
t.Fatalf("got %d items, want 2", len(got))
}

// svc_a has service labels
wantA := []*prompb.Label{
{Name: "__name__", Value: "test_counter"},
{Name: "service", Value: "svc_a"},
{Name: "team", Value: "backend"},
}
if !reflect.DeepEqual(got[0].Labels, wantA) {
t.Errorf("svc_a: got labels %+v, want %+v", got[0].Labels, wantA)
}

// svc_b does not
wantB := []*prompb.Label{
{Name: "__name__", Value: "test_counter"},
{Name: "service", Value: "svc_b"},
}
if !reflect.DeepEqual(got[1].Labels, wantB) {
t.Errorf("svc_b: got labels %+v, want %+v", got[1].Labels, wantB)
}
})
}
70 changes: 70 additions & 0 deletions runtimes/go/metrics/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,76 @@ func BenchmarkCounter_NewLabelSometimes(b *testing.B) {
}
}

func TestRegisterServiceLabels(t *testing.T) {
rt := reqtrack.New(zerolog.Logger{}, nil, nil)
reg := NewRegistry(rt, 2)

t.Run("stores and retrieves labels", func(t *testing.T) {
reg.RegisterServiceLabels("svc1", map[string]string{"team": "backend", "env": "prod"})
got := reg.ServiceLabels()
want := map[string][]KeyValue{
"svc1": {{Key: "env", Value: "prod"}, {Key: "team", Value: "backend"}},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("got %+v, want %+v", got, want)
}
})

t.Run("filters reserved keys", func(t *testing.T) {
reg.RegisterServiceLabels("svc2", map[string]string{
"service": "evil",
"__name__": "evil",
"endpoint": "evil",
"code": "evil",
"team": "ok",
})
got := reg.ServiceLabels()["svc2"]
want := []KeyValue{{Key: "team", Value: "ok"}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("got %+v, want %+v", got, want)
}
})

t.Run("filters empty keys and values", func(t *testing.T) {
reg.RegisterServiceLabels("svc3", map[string]string{
"": "no-key",
"valid": "",
"good": "value",
})
got := reg.ServiceLabels()["svc3"]
want := []KeyValue{{Key: "good", Value: "value"}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("got %+v, want %+v", got, want)
}
})

t.Run("deletes on empty input", func(t *testing.T) {
reg.RegisterServiceLabels("svc4", map[string]string{"team": "x"})
if reg.ServiceLabels()["svc4"] == nil {
t.Fatal("expected labels to be set")
}
reg.RegisterServiceLabels("svc4", nil)
if reg.ServiceLabels() != nil && reg.ServiceLabels()["svc4"] != nil {
t.Fatal("expected labels to be deleted")
}
})

t.Run("deletes when all keys reserved", func(t *testing.T) {
reg.RegisterServiceLabels("svc5", map[string]string{"team": "x"})
reg.RegisterServiceLabels("svc5", map[string]string{"service": "evil"})
if labels := reg.ServiceLabels(); labels != nil && labels["svc5"] != nil {
t.Fatalf("expected labels to be deleted, got %+v", labels["svc5"])
}
})

t.Run("returns nil when no labels registered", func(t *testing.T) {
emptyReg := NewRegistry(rt, 1)
if got := emptyReg.ServiceLabels(); got != nil {
t.Fatalf("expected nil, got %+v", got)
}
})
}

func eq[Val comparable](t testing.TB, got, want Val) {
t.Helper()
if got != want {
Expand Down
55 changes: 55 additions & 0 deletions runtimes/go/metrics/registry_internal.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package metrics

import (
"fmt"
"sort"
"sync"
"sync/atomic"

Expand All @@ -14,6 +15,8 @@ type Registry struct {
numSvcs uint16
tsid uint64
registry sync.Map // map[registryKey]*timeseries

serviceLabels sync.Map // map[string][]KeyValue — extra labels per service name
}

func NewRegistry(rt *reqtrack.RequestTracker, numServicesInBinary int) *Registry {
Expand Down Expand Up @@ -84,6 +87,11 @@ type CollectedMetric struct {
Labels []KeyValue
Val any // []T where T is any of Value
Valid []atomic.Bool

// ServiceLabels maps service names to additional labels that should be
// included when exporting this metric for that service. It is nil when
// no service labels have been registered.
ServiceLabels map[string][]KeyValue
}

type registryKey struct {
Expand Down Expand Up @@ -122,3 +130,50 @@ func getTS[T any](r *Registry, name string, labels any, info MetricInfo) (ts *ti
})
return val.(*timeseries[T]), loaded
}

// reservedLabelKeys are label keys that are set by the exporters and must
// not be overwritten by user-provided service labels.
var reservedLabelKeys = map[string]bool{
"__name__": true,
"service": true,
"endpoint": true,
"code": true,
}

// RegisterServiceLabels registers additional labels to be included with all
// metrics exported for the named service. This is useful for enriching built-in
// metrics (like e_requests_total) with custom metadata for alert routing or
// dashboard filtering.
//
// Labels with reserved keys (service, endpoint, code, __name__) or empty
// values are silently skipped. Subsequent calls for the same service replace
// previous labels. Passing nil or empty labels removes any previously
// registered labels for the service.
func (r *Registry) RegisterServiceLabels(serviceName string, labels map[string]string) {
kvs := make([]KeyValue, 0, len(labels))
for k, v := range labels {
if k == "" || v == "" || reservedLabelKeys[k] {
continue
}
kvs = append(kvs, KeyValue{Key: k, Value: v})
}
if len(kvs) == 0 {
r.serviceLabels.Delete(serviceName)
return
}
sort.Slice(kvs, func(i, j int) bool { return kvs[i].Key < kvs[j].Key })
r.serviceLabels.Store(serviceName, kvs)
}

// ServiceLabels returns a snapshot of all registered service labels.
func (r *Registry) ServiceLabels() map[string][]KeyValue {
result := make(map[string][]KeyValue)
r.serviceLabels.Range(func(key, value any) bool {
result[key.(string)] = value.([]KeyValue)
return true
})
if len(result) == 0 {
return nil
}
return result
}
Loading