Chaos Engineering

var builder = new ResiliencePipelineBuilder<HttpResponseMessage>();

// First, configure regular resilience strategies
builder
    .AddConcurrencyLimiter(10, 100)
    .AddRetry(new RetryStrategyOptions<HttpResponseMessage> { /* configure options */ })
    .AddCircuitBreaker(new CircuitBreakerStrategyOptions<HttpResponseMessage> { /* configure options */ })
    .AddTimeout(TimeSpan.FromSeconds(5));

// Finally, configure chaos strategies if you want to inject chaos.
// These should come after the regular resilience strategies.

// 2% of all requests will be injected with chaos fault.
const double FaultInjectionRate = 0.02;
// For the remaining 98% of total requests, 50% of them will be injected with latency. Then 49% of total request will be injected with chaos latency.
// Latency injection does not return early.
const double LatencyInjectionRate = 0.50;
// For the remaining 98% of total requests, 10% of them will be injected with outcome. Then 9.8% of total request will be injected with chaos outcome.
const double OutcomeInjectionRate = 0.10;
// For the remaining 88.2% of total requests, 1% of them will be injected with behavior. Then 0.882% of total request will be injected with chaos behavior.
const double BehaviorInjectionRate = 0.01;

builder
    .AddChaosFault(FaultInjectionRate, () => new InvalidOperationException("Injected by chaos strategy!")) // Inject a chaos fault to executions
    .AddChaosLatency(LatencyInjectionRate, TimeSpan.FromMinutes(1)) // Inject a chaos latency to executions
    .AddChaosOutcome(OutcomeInjectionRate, () => new HttpResponseMessage(System.Net.HttpStatusCode.InternalServerError)) // Inject a chaos outcome to executions
    .AddChaosBehavior(BehaviorInjectionRate, cancellationToken => RestartRedisAsync(cancellationToken)); // Inject a chaos behavior to executions

services.AddResiliencePipeline("chaos-pipeline", (builder, context) =>
{
    var environment = context.ServiceProvider.GetRequiredService<IHostEnvironment>();

    builder.AddChaosFault(new ChaosFaultStrategyOptions
    {
        EnabledGenerator = args =>
        {
            // Enable chaos in development and staging environments.
            if (environment.IsDevelopment() || environment.IsStaging())
            {
                return ValueTask.FromResult(true);
            }

            // Enable chaos for specific users or tenants, even in production environments.
            if (ShouldEnableChaos(args.Context))
            {
                return ValueTask.FromResult(true);
            }

            return ValueTask.FromResult(false);
        },
        InjectionRateGenerator = args =>
        {
            if (environment.IsStaging())
            {
                // 1% chance of failure on staging environments.
                return ValueTask.FromResult(0.01);
            }

            if (environment.IsDevelopment())
            {
                // 5% chance of failure on development environments.
                return ValueTask.FromResult(0.05);
            }

            // The context can carry information to help determine the injection rate.
            // For instance, in production environments, you might have certain test users or tenants
            // for whom you wish to inject chaos.
            if (ResolveInjectionRate(args.Context, out double injectionRate))
            {
                return ValueTask.FromResult(injectionRate);
            }

            // No chaos on production environments.
            return ValueTask.FromResult(0.0);
        },
        FaultGenerator = new FaultGenerator()
            .AddException<TimeoutException>()
            .AddException<HttpRequestException>()
    });
});

public interface IChaosManager
{
    ValueTask<bool> IsChaosEnabled(ResilienceContext context);

    ValueTask<double> GetInjectionRate(ResilienceContext context);
}

services.AddResiliencePipeline("chaos-pipeline", (builder, context) =>
{
    var chaosManager = context.ServiceProvider.GetRequiredService<IChaosManager>();

    builder
        .AddChaosFault(new ChaosFaultStrategyOptions
        {
            EnabledGenerator = args => chaosManager.IsChaosEnabled(args.Context),
            InjectionRateGenerator = args => chaosManager.GetInjectionRate(args.Context),
            FaultGenerator = new FaultGenerator()
                .AddException<TimeoutException>()
                .AddException<HttpRequestException>()
        })
        .AddChaosLatency(new ChaosLatencyStrategyOptions
        {
            EnabledGenerator = args => chaosManager.IsChaosEnabled(args.Context),
            InjectionRateGenerator = args => chaosManager.GetInjectionRate(args.Context),
            Latency = TimeSpan.FromSeconds(60)
        });
});