Distributed Tracing

What is Distributed Tracing?

Distributed tracing tracks requests as they flow through multiple services, helping you understand performance bottlenecks and debug issues in microservices.

OpenTelemetry

dotnet add package OpenTelemetry.Extensions.Hosting
dotnet add package OpenTelemetry.Instrumentation.AspNetCore
dotnet add package OpenTelemetry.Instrumentation.Http
dotnet add package OpenTelemetry.Exporter.Jaeger

using OpenTelemetry.Resources;
using OpenTelemetry.Trace;

builder.Services.AddOpenTelemetry()
    .WithTracing(tracerProviderBuilder =>
    {
        tracerProviderBuilder
            .AddSource("InvenTrack")
            .SetResourceBuilder(
                ResourceBuilder.CreateDefault()
                    .AddService("InvenTrack"))
            .AddAspNetCoreInstrumentation()
            .AddHttpClientInstrumentation()
            .AddJaegerExporter(options =>
            {
                options.AgentHost = "localhost";
                options.AgentPort = 6831;
            });
    });

Run Jaeger with Docker

docker run -d --name jaeger \
  -p 6831:6831/udp \
  -p 16686:16686 \
  jaegertracing/all-in-one:latest

Access Jaeger UI at: http://localhost:16686

Custom Activity

public static class Telemetry
{
    public static readonly ActivitySource ActivitySource = 
        new ActivitySource("InvenTrack");
}

public class ProductService
{
    public async Task<Product> GetProductAsync(int id)
    {
        using (var activity = Telemetry.ActivitySource.StartActivity("GetProduct"))
        {
            activity?.SetTag("product.id", id);
            
            var product = await _context.Products.FindAsync(id);
            
            activity?.SetTag("product.name", product?.Name);
            
            return product;
        }
    }
}

Correlation IDs

public class CorrelationIdMiddleware
{
    private readonly RequestDelegate _next;
    private const string CorrelationIdHeader = "X-Correlation-ID";

    public async Task InvokeAsync(HttpContext context)
    {
        var correlationId = context.Request.Headers[CorrelationIdHeader].FirstOrDefault() 
            ?? Guid.NewGuid().ToString();

        context.Response.Headers.Add(CorrelationIdHeader, correlationId);
        context.Items["CorrelationId"] = correlationId;

        await _next(context);
    }
}

app.UseMiddleware<CorrelationIdMiddleware>();

InvenTrack Example

public class OrderService
{
    public async Task CreateOrderAsync(Order order)
    {
        using (var activity = Telemetry.ActivitySource.StartActivity("CreateOrder"))
        {
            activity?.SetTag("order.id", order.Id);
            activity?.SetTag("order.total", order.Total);
            
            // Validate stock
            using (var stockActivity = Telemetry.ActivitySource.StartActivity("ValidateStock"))
            {
                await ValidateStockAsync(order);
            }
            
            // Save order
            await _context.Orders.AddAsync(order);
            await _context.SaveChangesAsync();
        }
    }
}

Key Takeaways

• Distributed tracing: Track requests across services
• OpenTelemetry: Standard for observability
• Jaeger: Tracing backend and UI
• ActivitySource: Create custom spans
• SetTag(): Add metadata to spans
• Correlation IDs: Track requests across services