Null values in trace fields using opentelemetry-dotnet client

[AmythD]

Hello,

I am observing null values in the Date field for traces recieved from OTEL agent for .NET causing dates to show up as Invalid on OpenSearch

[
  {
    "_index": "otel-v1-apm-span-000001",
    "_id": "989fe760834aadaa",
    "_score": 2.717529,
    "_source": {
      "traceId": "1cb92f0264995e7024267bdea3bdaeaa",
      "droppedLinksCount": 0,
      "kind": "SPAN_KIND_CONSUMER",
      "droppedEventsCount": 0,
      "traceGroupFields": {
        "endTime": null,
        "durationInNanos": null,
        "statusCode": null
      },
      "traceGroup": null,
      "serviceName": "Wave.Api",
      "parentSpanId": "c6109393a176a7a1",
      "spanId": "989fe760834aadaa",
      "traceState": "",
      "name": "Wave.Service.CommandHandlers.DraftWave.DraftWaveCommandRequest receive",
      "startTime": "2024-07-30T17:01:59.571839200Z",
      "links": [],
      "endTime": "2024-07-30T17:01:59.665671200Z",
      "droppedAttributesCount": 0,
      "durationInNanos": 93832000,
      "events": [],
      "span.attributes.App-Session-Id": "",
      "span.attributes.User-Agent": "curl/7.76.1",
      "resource.attributes.telemetry@sdk@language": "dotnet",
      "span.attributes.Machine": "api-wave-2024-30-5785cc5b44-ssm6j",
      "span.attributes.App-User-Id": "apisixtest",
      "span.attributes.App-WareHouse-Id": "",
      "span.attributes.CorrelationId": "01J3JKP0MS0JGHB16V5BSC4VJY",
      "resource.attributes.service@instance@id": "8ce4fd1d-f534-46d5-aebc-04dc8ccc3a97",
      "resource.attributes.telemetry@sdk@version": "1.7.0",
      "resource.attributes.service@name": "Wave.Api",
      "status.code": 0,
      "instrumentationScope.name": "ActivityTracer",
      "span.attributes.App-Environment": "Environment1",
      "resource.attributes.telemetry@sdk@name": "opentelemetry",
      "span.attributes.App-Account-Id": "PA01",
      "span.attributes.App-BU-Id": "1001"
    }
  },
  {
    "_index": "otel-v1-apm-span-000001",
    "_id": "c6109393a176a7a1",
    "_score": 2.717529,
    "_source": {
      "traceId": "1cb92f0264995e7024267bdea3bdaeaa",
      "droppedLinksCount": 0,
      "kind": "SPAN_KIND_SERVER",
      "droppedEventsCount": 0,
      "traceGroupFields": {
        "endTime": null,
        "durationInNanos": null,
        "statusCode": null
      },
      "traceGroup": null,
      "serviceName": "Wave.Api",
      "parentSpanId": "beb909d86843f0cd",
      "spanId": "c6109393a176a7a1",
      "traceState": "",
      "name": "POST wave",
      "startTime": "2024-07-30T17:01:59.563415900Z",
      "links": [],
      "endTime": "2024-07-30T17:01:59.668281Z",
      "droppedAttributesCount": 0,
      "durationInNanos": 104865100,
      "events": [],
      "span.attributes.User-Agent": "curl/7.76.1",
      "span.attributes.ApplicationVersion": "2024-30",
      "span.attributes.Machine": "api-wave-2024-30-5785cc5b44-ssm6j",
      "span.attributes.CorrelationId": "",
      "resource.attributes.service@instance@id": "8ce4fd1d-f534-46d5-aebc-04dc8ccc3a97",
      "resource.attributes.telemetry@sdk@version": "1.7.0",
      "resource.attributes.service@name": "Wave.Api",
      "status.code": 0,
      "span.attributes.http@response@status_code": 200,
      "span.attributes.url@scheme": "https",
      "span.attributes.App-Account-Id": "PA01",
      "span.attributes.url@path": "/2024-30/api/wave",
      "span.attributes.App-BU-Id": "1001",
      "span.attributes.App-Session-Id": "",
      "resource.attributes.telemetry@sdk@language": "dotnet",
      "span.attributes.http@request@method": "POST",
      "span.attributes.App-User-Id": "apisixtest",
      "span.attributes.App-WareHouse-Id": "",
      "instrumentationScope.name": "Microsoft.AspNetCore",
      "span.attributes.user_agent@original": "curl/7.76.1",
      "span.attributes.App-Environment": "Environment1",
      "resource.attributes.telemetry@sdk@name": "opentelemetry",
      "span.attributes.http@route": "wave",
      "span.attributes.network@protocol@version": "1.1",
      "span.attributes.server@address": "api-wave.wms.svc.cluster.local"
    }
  }, 

My setup includes OTEL client -> OTEL collector -> DataPrepper -> OpenSearch

OTEL collector configuration:

extensions:
health_check:

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
  prometheus:
    config:
      scrape_configs:
      - job_name: 'otel-collector'
        scrape_interval: 10s
        static_configs:
        - targets: ['0.0.0.0:8888']

processors:
  batch:

exporters:
  debug:
    verbosity: detailed
    sampling_initial: 5
    sampling_thereafter: 200
  otlp/data-prepper:
    endpoint: 'X.X.X.X:21890'
    tls:
      insecure: true

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [debug,otlp/data-prepper]
  extensions: [health_check]

DataPrepper Configuration:

otel-trace-pipeline:
  workers: 8
  delay: "100"
  source:
    otel_trace_source:
      ssl: false
      authentication:
        unauthenticated:
  buffer:
    bounded_blocking:
       buffer_size: 25600
       batch_size: 400
  sink:
    - pipeline:
        name: "raw-pipeline"
    - pipeline:
        name: "service-map-pipeline"
raw-pipeline:
  workers: 8
  delay: "3000"
  source:
    pipeline:
      name: "otel-trace-pipeline"
  buffer:
      bounded_blocking:
         buffer_size: 25600
         batch_size: 3200
  processor:
    - otel_trace_raw:
    - otel_trace_group:
        hosts: [ "https://opensearch:9200" ]
        username: "admin"
        password: "admin"
  sink:
    - opensearch:
        hosts: [ "https://opensearch:9200" ]
        index_type: trace-analytics-raw
        username: "admin"
        password: "admin"

service-map-pipeline:
  workers: 8
  delay: "100"
  source:
    pipeline:
      name: "otel-trace-pipeline"
  processor:
    - service_map_stateful:
        window_duration: 180
  buffer:
      bounded_blocking:
         buffer_size: 25600
         batch_size: 400
  sink:
    - opensearch:
        hosts: [ "https://opensearch:9200" ]
        index_type: trace-analytics-service-map
        username: "admin"
        password: "admin"

Any ideas on what could be wrong here? Thanks

[cijothomas]

I am observing null values in the Date field for traces

Can you a repro, or the output from the Collector's debug output, so we can isolate the issue better.

[AmythD]

@cijothomas , It seems like the issue was that "traceGroup" field was not properly set on the spans. Traces with valid "traceGroup" value is showing correct date/time.