diff --git a/docs/ngc-managed/cluster-management/configuration.md b/docs/ngc-managed/cluster-management/configuration.md index 0749820a89..ce8a9a62f0 100644 --- a/docs/ngc-managed/cluster-management/configuration.md +++ b/docs/ngc-managed/cluster-management/configuration.md @@ -788,10 +788,13 @@ BYOO metric subset example: - 'metric.name != "BpsInstrument"' byooWorkloadMetrics: dropLabels: - - metric_subset_enabled - custom_label ``` +When `byooMetricSubset.enabled` is true, `dropLabels` extends the default +`metric_subset_enabled` label. The configured labels are removed from both the +primary metrics pipeline and the metric subset endpoint on port `19091`. + **Apply via Helm:** ```bash diff --git a/docs/user/cluster-management/configuration.md b/docs/user/cluster-management/configuration.md index be3f81e5d8..1b309466af 100644 --- a/docs/user/cluster-management/configuration.md +++ b/docs/user/cluster-management/configuration.md @@ -918,10 +918,13 @@ agentConfig: - 'metric.name != "BpsInstrument"' byooWorkloadMetrics: dropLabels: - - metric_subset_enabled - custom_label ``` +When `byooMetricSubset.enabled` is true, `dropLabels` extends the default +`metric_subset_enabled` label. The configured labels are removed from both the +primary metrics pipeline and the metric subset endpoint on port `19091`. + Apply via Helm: ```bash diff --git a/src/compute-plane-services/byoo-otel-collector/README.md b/src/compute-plane-services/byoo-otel-collector/README.md index 749de5d73e..89d50ec7fa 100644 --- a/src/compute-plane-services/byoo-otel-collector/README.md +++ b/src/compute-plane-services/byoo-otel-collector/README.md @@ -141,7 +141,7 @@ Chunking is disabled by default. Configure it with: - `BYOO_OTEL_COLLECTOR_CONFIG_B64`: optional base64-encoded JSON for advanced collector rendering overrides, such as exporterhelper timeout, retry, sending queue, sending queue batch, memory limiter, batch, log batch, and separate log and trace sampler settings. Both samplers support sampling percentage, mode, hash seed, and fail closed. The log sampler also supports attribute source, source attribute, and sampling priority. - `BYOO_METRIC_SUBSET_ENABLED`: enables an additional OTLP-only metrics pipeline that exposes filtered user metrics through a Prometheus exporter on port `19091`. Disabled by default. - `BYOO_METRIC_SUBSET_FILTER_CONFIG`: optional YAML filter processor config for the metric subset pipeline. If unset, the default drops every metric except `BpsInstrument`, `FpsInstrument`, `RtdInstrument`, and `StageOpenDuration`, and drops datapoints/resources explicitly labeled `metric_subset_enabled=false`. -- `BYOO_WORKLOAD_METRICS_DROP_LABELS`: comma-separated resource attribute names removed from the generated workload `metrics` pipeline. If unset, defaults to `metric_subset_enabled` only when the metric subset pipeline is enabled. +- `BYOO_WORKLOAD_METRICS_DROP_LABELS`: comma-separated resource attribute names removed from the generated workload metrics pipelines. When the metric subset pipeline is enabled, configured labels extend the default `metric_subset_enabled` label. Labels are removed from both the primary and metric subset pipelines. When chunking is enabled, each emitted chunk preserves the original log metadata and adds these attributes so chunks can be grouped in the backend: diff --git a/src/compute-plane-services/byoo-otel-collector/examples/otelconfigs/k8s/config_function_container_metric_subset.yaml b/src/compute-plane-services/byoo-otel-collector/examples/otelconfigs/k8s/config_function_container_metric_subset.yaml index 705c779984..5642a50716 100644 --- a/src/compute-plane-services/byoo-otel-collector/examples/otelconfigs/k8s/config_function_container_metric_subset.yaml +++ b/src/compute-plane-services/byoo-otel-collector/examples/otelconfigs/k8s/config_function_container_metric_subset.yaml @@ -285,6 +285,7 @@ service: - memory_limiter - filter/metric_subset - resource + - resource/workload_metrics_drop_labels - metrics_transform - batch/metric_subset extensions: diff --git a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/otelconfig_test.go b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/otelconfig_test.go index 6e7ede922c..771cec7d13 100644 --- a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/otelconfig_test.go +++ b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/otelconfig_test.go @@ -212,7 +212,7 @@ func TestGetTemplateConfig(t *testing.T) { "NVCF_ZONE_NAME": "zone-1", "BYOO_METRIC_SUBSET_ENABLED": "true", "BYOO_METRIC_SUBSET_FILTER_CONFIG": "error_mode: ignore\nmetric_conditions:\n - 'metric.name == \"drop\"'\n", - "BYOO_WORKLOAD_METRICS_DROP_LABELS": "metric_subset_enabled, custom_label, metric_subset_enabled", + "BYOO_WORKLOAD_METRICS_DROP_LABELS": "custom_label", }, expectErr: false, expect: func(t *testing.T, cfg TemplateConfig) { diff --git a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go index 60983fa7b1..34501e3fc0 100644 --- a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go +++ b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go @@ -248,6 +248,12 @@ func resolvedWorkloadMetricsDropLabels(configured string, metricSubsetEnabled bo seen := map[string]struct{}{} labels := []string{} + if metricSubsetEnabled { + for _, label := range defaultWorkloadMetricsDropLabels { + seen[label] = struct{}{} + labels = append(labels, label) + } + } for _, label := range strings.Split(configured, ",") { label = strings.TrimSpace(label) if label == "" { @@ -578,7 +584,7 @@ func cloneConfigValue(value interface{}) interface{} { } } -func addMetricSubsetPipeline(otelConfig *OpenTelemetryConfig, config MetricSubsetConfig) { +func addMetricSubsetPipeline(otelConfig *OpenTelemetryConfig, config MetricSubsetConfig, workloadMetricsDropLabelsProcessor string) { addMetricSubsetExporter(otelConfig) filterConfig := config.FilterConfig @@ -604,9 +610,14 @@ func addMetricSubsetPipeline(otelConfig *OpenTelemetryConfig, config MetricSubse "memory_limiter", metricSubsetFilterProcessorID, "resource", + } + if workloadMetricsDropLabelsProcessor != "" { + metricSubsetPipeline.Processors = append(metricSubsetPipeline.Processors, workloadMetricsDropLabelsProcessor) + } + metricSubsetPipeline.Processors = append(metricSubsetPipeline.Processors, "metrics_transform", metricSubsetBatchProcessorID, - } + ) otelConfig.Service.Pipelines["metrics/metric_subset"] = metricSubsetPipeline } @@ -888,14 +899,15 @@ func generateExportersAndService(config TelemetryConfig, otelConfig *OpenTelemet metricPipeline.Receivers = []string{"otlp", "prometheus"} metricPipeline.Exporters = []string{exporterId} metricPipeline.Processors = []string{"memory_limiter", "filter/metrics", "resource"} - if processorID := addWorkloadMetricsDropLabelsProcessor(otelConfig, tmplConfig.WorkloadMetrics.DropLabels); processorID != "" { - metricPipeline.Processors = append(metricPipeline.Processors, processorID) + workloadMetricsDropLabelsProcessor := addWorkloadMetricsDropLabelsProcessor(otelConfig, tmplConfig.WorkloadMetrics.DropLabels) + if workloadMetricsDropLabelsProcessor != "" { + metricPipeline.Processors = append(metricPipeline.Processors, workloadMetricsDropLabelsProcessor) } metricPipeline.Processors = append(metricPipeline.Processors, "metrics_transform", "batch") otelConfig.Service.Pipelines["metrics"] = metricPipeline if tmplConfig.MetricSubset.Enabled { - addMetricSubsetPipeline(otelConfig, tmplConfig.MetricSubset) + addMetricSubsetPipeline(otelConfig, tmplConfig.MetricSubset, workloadMetricsDropLabelsProcessor) } } diff --git a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go index 5ccb317551..401747522f 100644 --- a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go +++ b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go @@ -159,6 +159,7 @@ func TestRenderOtelConfigWithMetricSubsetPipeline(t *testing.T) { "memory_limiter", metricSubsetFilterProcessorID, "resource", + workloadMetricsDropLabelsProcessorID, "metrics_transform", metricSubsetBatchProcessorID, }, otelConfig.Service.Pipelines["metrics/metric_subset"].Processors) @@ -856,7 +857,7 @@ func TestGenerateExportersAndServiceAddsMetricSubsetPipeline(t *testing.T) { FilterConfig: filterConfig, }, WorkloadMetrics: WorkloadMetricsConfig{ - DropLabels: []string{"metric_subset_enabled"}, + DropLabels: []string{"metric_subset_enabled", "custom_label"}, }, }) @@ -888,6 +889,10 @@ func TestGenerateExportersAndServiceAddsMetricSubsetPipeline(t *testing.T) { "key": "metric_subset_enabled", "action": "delete", }, + { + "key": "custom_label", + "action": "delete", + }, }, }, otelConfig.Processors[workloadMetricsDropLabelsProcessorID]) @@ -898,6 +903,7 @@ func TestGenerateExportersAndServiceAddsMetricSubsetPipeline(t *testing.T) { "memory_limiter", metricSubsetFilterProcessorID, "resource", + workloadMetricsDropLabelsProcessorID, "metrics_transform", metricSubsetBatchProcessorID, }, metricSubsetPipeline.Processors)