✨ Gateway readiness check
This commit is contained in:
9
DysonNetwork.Gateway/Health/GatewayConstant.cs
Normal file
9
DysonNetwork.Gateway/Health/GatewayConstant.cs
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
namespace DysonNetwork.Gateway.Health;
|
||||||
|
|
||||||
|
public abstract class GatewayConstant
|
||||||
|
{
|
||||||
|
public static readonly string[] ServiceNames = ["ring", "pass", "drive", "sphere", "develop", "insight", "zone"];
|
||||||
|
|
||||||
|
// Core services stands with w/o these services the functional of entire app will broke.
|
||||||
|
public static readonly string[] CoreServiceNames = ["ring", "pass", "drive", "sphere"];
|
||||||
|
}
|
||||||
60
DysonNetwork.Gateway/Health/GatewayHealthAggregator.cs
Normal file
60
DysonNetwork.Gateway/Health/GatewayHealthAggregator.cs
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
using NodaTime;
|
||||||
|
|
||||||
|
namespace DysonNetwork.Gateway.Health;
|
||||||
|
|
||||||
|
public class GatewayHealthAggregator(IHttpClientFactory httpClientFactory, GatewayReadinessStore store)
|
||||||
|
: BackgroundService
|
||||||
|
{
|
||||||
|
private async Task<ServiceHealthState> CheckService(string serviceName)
|
||||||
|
{
|
||||||
|
var client = httpClientFactory.CreateClient("health");
|
||||||
|
var now = SystemClock.Instance.GetCurrentInstant();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Use the service discovery to lookup service
|
||||||
|
// The service defaults give every single service a health endpoint that we can use here
|
||||||
|
using var response = await client.GetAsync($"http://{serviceName}/health");
|
||||||
|
|
||||||
|
if (response.IsSuccessStatusCode)
|
||||||
|
{
|
||||||
|
return new ServiceHealthState(
|
||||||
|
serviceName,
|
||||||
|
true,
|
||||||
|
now,
|
||||||
|
null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new ServiceHealthState(
|
||||||
|
serviceName,
|
||||||
|
false,
|
||||||
|
now,
|
||||||
|
$"StatusCode: {(int)response.StatusCode}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
return new ServiceHealthState(
|
||||||
|
serviceName,
|
||||||
|
false,
|
||||||
|
now,
|
||||||
|
ex.Message
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
foreach (var service in GatewayConstant.ServiceNames)
|
||||||
|
{
|
||||||
|
var result = await CheckService(service);
|
||||||
|
store.Update(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
35
DysonNetwork.Gateway/Health/GatewayReadinessMiddleware.cs
Normal file
35
DysonNetwork.Gateway/Health/GatewayReadinessMiddleware.cs
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
namespace DysonNetwork.Gateway.Health;
|
||||||
|
|
||||||
|
using Microsoft.AspNetCore.Http;
|
||||||
|
|
||||||
|
public sealed class GatewayReadinessMiddleware(RequestDelegate next)
|
||||||
|
{
|
||||||
|
public async Task InvokeAsync(HttpContext context, GatewayReadinessStore store)
|
||||||
|
{
|
||||||
|
if (context.Request.Path.StartsWithSegments("/health"))
|
||||||
|
{
|
||||||
|
await next(context);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var readiness = store.Current;
|
||||||
|
|
||||||
|
// Only core services participate in readiness gating
|
||||||
|
var notReadyCoreServices = readiness.Services
|
||||||
|
.Where(kv => GatewayConstant.CoreServiceNames.Contains(kv.Key))
|
||||||
|
.Where(kv => !kv.Value.IsHealthy)
|
||||||
|
.Select(kv => kv.Key)
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
|
if (notReadyCoreServices.Length > 0)
|
||||||
|
{
|
||||||
|
context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
|
||||||
|
var unavailableServices = string.Join(", ", notReadyCoreServices);
|
||||||
|
context.Response.Headers["X-NotReady-Services"] = unavailableServices;
|
||||||
|
await context.Response.WriteAsync("Solar Network is warming up. Try again later please.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await next(context);
|
||||||
|
}
|
||||||
|
}
|
||||||
76
DysonNetwork.Gateway/Health/GatewayReadinessStore.cs
Normal file
76
DysonNetwork.Gateway/Health/GatewayReadinessStore.cs
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
using NodaTime;
|
||||||
|
|
||||||
|
namespace DysonNetwork.Gateway.Health;
|
||||||
|
|
||||||
|
public record ServiceHealthState(
|
||||||
|
string ServiceName,
|
||||||
|
bool IsHealthy,
|
||||||
|
Instant LastChecked,
|
||||||
|
string? Error
|
||||||
|
);
|
||||||
|
|
||||||
|
public record GatewayReadinessState(
|
||||||
|
bool IsReady,
|
||||||
|
IReadOnlyDictionary<string, ServiceHealthState> Services,
|
||||||
|
Instant LastUpdated
|
||||||
|
);
|
||||||
|
|
||||||
|
public class GatewayReadinessStore
|
||||||
|
{
|
||||||
|
private readonly Lock _lock = new();
|
||||||
|
|
||||||
|
private readonly Dictionary<string, ServiceHealthState> _services = new();
|
||||||
|
|
||||||
|
public GatewayReadinessState Current { get; private set; } = new(
|
||||||
|
IsReady: false,
|
||||||
|
Services: new Dictionary<string, ServiceHealthState>(),
|
||||||
|
LastUpdated: SystemClock.Instance.GetCurrentInstant()
|
||||||
|
);
|
||||||
|
|
||||||
|
public IReadOnlyCollection<string> ServiceNames => _services.Keys;
|
||||||
|
|
||||||
|
public GatewayReadinessStore()
|
||||||
|
{
|
||||||
|
InitializeServices(GatewayConstant.ServiceNames);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void InitializeServices(IEnumerable<string> serviceNames)
|
||||||
|
{
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
_services.Clear();
|
||||||
|
|
||||||
|
foreach (var name in serviceNames)
|
||||||
|
{
|
||||||
|
_services[name] = new ServiceHealthState(
|
||||||
|
name,
|
||||||
|
IsHealthy: false,
|
||||||
|
LastChecked: SystemClock.Instance.GetCurrentInstant(),
|
||||||
|
Error: "Not checked yet"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
RecalculateLocked();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Update(ServiceHealthState state)
|
||||||
|
{
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
_services[state.ServiceName] = state;
|
||||||
|
RecalculateLocked();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void RecalculateLocked()
|
||||||
|
{
|
||||||
|
var isReady = _services.Count > 0 && _services.Values.All(s => s.IsHealthy);
|
||||||
|
|
||||||
|
Current = new GatewayReadinessState(
|
||||||
|
IsReady: isReady,
|
||||||
|
Services: new Dictionary<string, ServiceHealthState>(_services),
|
||||||
|
LastUpdated: SystemClock.Instance.GetCurrentInstant()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
14
DysonNetwork.Gateway/Health/GatewayStatusController.cs
Normal file
14
DysonNetwork.Gateway/Health/GatewayStatusController.cs
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
|
||||||
|
namespace DysonNetwork.Gateway.Health;
|
||||||
|
|
||||||
|
[ApiController]
|
||||||
|
[Route("/health")]
|
||||||
|
public class GatewayStatusController(GatewayReadinessStore readinessStore) : ControllerBase
|
||||||
|
{
|
||||||
|
[HttpGet]
|
||||||
|
public ActionResult<GatewayReadinessState> GetHealthStatus()
|
||||||
|
{
|
||||||
|
return Ok(readinessStore.Current);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,12 @@
|
|||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
using System.Threading.RateLimiting;
|
using System.Threading.RateLimiting;
|
||||||
|
using DysonNetwork.Gateway.Health;
|
||||||
using DysonNetwork.Shared.Http;
|
using DysonNetwork.Shared.Http;
|
||||||
using Yarp.ReverseProxy.Configuration;
|
using Yarp.ReverseProxy.Configuration;
|
||||||
using Microsoft.AspNetCore.HttpOverrides;
|
using Microsoft.AspNetCore.HttpOverrides;
|
||||||
|
using NodaTime;
|
||||||
|
using NodaTime.Serialization.SystemTextJson;
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
@@ -9,17 +14,19 @@ builder.AddServiceDefaults();
|
|||||||
|
|
||||||
builder.ConfigureAppKestrel(builder.Configuration, maxRequestBodySize: long.MaxValue, enableGrpc: false);
|
builder.ConfigureAppKestrel(builder.Configuration, maxRequestBodySize: long.MaxValue, enableGrpc: false);
|
||||||
|
|
||||||
|
builder.Services.AddSingleton<GatewayReadinessStore>();
|
||||||
|
builder.Services.AddHostedService<GatewayHealthAggregator>();
|
||||||
|
|
||||||
builder.Services.AddCors(options =>
|
builder.Services.AddCors(options =>
|
||||||
{
|
{
|
||||||
options.AddDefaultPolicy(
|
options.AddDefaultPolicy(policy =>
|
||||||
policy =>
|
{
|
||||||
{
|
policy.SetIsOriginAllowed(origin => true)
|
||||||
policy.SetIsOriginAllowed(origin => true)
|
.AllowAnyMethod()
|
||||||
.AllowAnyMethod()
|
.AllowAnyHeader()
|
||||||
.AllowAnyHeader()
|
.AllowCredentials()
|
||||||
.AllowCredentials()
|
.WithExposedHeaders("X-Total");
|
||||||
.WithExposedHeaders("X-Total");
|
});
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
builder.Services.AddRateLimiter(options =>
|
builder.Services.AddRateLimiter(options =>
|
||||||
@@ -40,23 +47,22 @@ builder.Services.AddRateLimiter(options =>
|
|||||||
});
|
});
|
||||||
|
|
||||||
options.OnRejected = async (context, token) =>
|
options.OnRejected = async (context, token) =>
|
||||||
{
|
{
|
||||||
// Log the rejected IP
|
// Log the rejected IP
|
||||||
var logger = context.HttpContext.RequestServices
|
var logger = context.HttpContext.RequestServices
|
||||||
.GetRequiredService<ILoggerFactory>()
|
.GetRequiredService<ILoggerFactory>()
|
||||||
.CreateLogger("RateLimiter");
|
.CreateLogger("RateLimiter");
|
||||||
|
|
||||||
var ip = context.HttpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown";
|
var ip = context.HttpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown";
|
||||||
logger.LogWarning("Rate limit exceeded for IP: {IP}", ip);
|
logger.LogWarning("Rate limit exceeded for IP: {IP}", ip);
|
||||||
|
|
||||||
// Respond to the client
|
// Respond to the client
|
||||||
context.HttpContext.Response.StatusCode = StatusCodes.Status429TooManyRequests;
|
context.HttpContext.Response.StatusCode = StatusCodes.Status429TooManyRequests;
|
||||||
await context.HttpContext.Response.WriteAsync(
|
await context.HttpContext.Response.WriteAsync(
|
||||||
"Rate limit exceeded. Try again later.", token);
|
"Rate limit exceeded. Try again later.", token);
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
var serviceNames = new[] { "ring", "pass", "drive", "sphere", "develop", "insight", "zone" };
|
|
||||||
|
|
||||||
var specialRoutes = new[]
|
var specialRoutes = new[]
|
||||||
{
|
{
|
||||||
@@ -86,7 +92,7 @@ var specialRoutes = new[]
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
var apiRoutes = serviceNames.Select(serviceName =>
|
var apiRoutes = GatewayConstant.ServiceNames.Select(serviceName =>
|
||||||
{
|
{
|
||||||
var apiPath = serviceName switch
|
var apiPath = serviceName switch
|
||||||
{
|
{
|
||||||
@@ -105,7 +111,7 @@ var apiRoutes = serviceNames.Select(serviceName =>
|
|||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
var swaggerRoutes = serviceNames.Select(serviceName => new RouteConfig
|
var swaggerRoutes = GatewayConstant.ServiceNames.Select(serviceName => new RouteConfig
|
||||||
{
|
{
|
||||||
RouteId = $"{serviceName}-swagger",
|
RouteId = $"{serviceName}-swagger",
|
||||||
ClusterId = serviceName,
|
ClusterId = serviceName,
|
||||||
@@ -119,7 +125,7 @@ var swaggerRoutes = serviceNames.Select(serviceName => new RouteConfig
|
|||||||
|
|
||||||
var routes = specialRoutes.Concat(apiRoutes).Concat(swaggerRoutes).ToArray();
|
var routes = specialRoutes.Concat(apiRoutes).Concat(swaggerRoutes).ToArray();
|
||||||
|
|
||||||
var clusters = serviceNames.Select(serviceName => new ClusterConfig
|
var clusters = GatewayConstant.ServiceNames.Select(serviceName => new ClusterConfig
|
||||||
{
|
{
|
||||||
ClusterId = serviceName,
|
ClusterId = serviceName,
|
||||||
HealthCheck = new HealthCheckConfig
|
HealthCheck = new HealthCheckConfig
|
||||||
@@ -147,7 +153,14 @@ builder.Services
|
|||||||
.LoadFromMemory(routes, clusters)
|
.LoadFromMemory(routes, clusters)
|
||||||
.AddServiceDiscoveryDestinationResolver();
|
.AddServiceDiscoveryDestinationResolver();
|
||||||
|
|
||||||
builder.Services.AddControllers();
|
builder.Services.AddControllers().AddJsonOptions(options =>
|
||||||
|
{
|
||||||
|
options.JsonSerializerOptions.NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals;
|
||||||
|
options.JsonSerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower;
|
||||||
|
options.JsonSerializerOptions.DictionaryKeyPolicy = JsonNamingPolicy.SnakeCaseLower;
|
||||||
|
|
||||||
|
options.JsonSerializerOptions.ConfigureForNodaTime(DateTimeZoneProviders.Tzdb);
|
||||||
|
});
|
||||||
|
|
||||||
var app = builder.Build();
|
var app = builder.Build();
|
||||||
|
|
||||||
@@ -155,14 +168,16 @@ var forwardedHeadersOptions = new ForwardedHeadersOptions
|
|||||||
{
|
{
|
||||||
ForwardedHeaders = ForwardedHeaders.All
|
ForwardedHeaders = ForwardedHeaders.All
|
||||||
};
|
};
|
||||||
forwardedHeadersOptions.KnownNetworks.Clear();
|
forwardedHeadersOptions.KnownIPNetworks.Clear();
|
||||||
forwardedHeadersOptions.KnownProxies.Clear();
|
forwardedHeadersOptions.KnownProxies.Clear();
|
||||||
app.UseForwardedHeaders(forwardedHeadersOptions);
|
app.UseForwardedHeaders(forwardedHeadersOptions);
|
||||||
|
|
||||||
app.UseCors();
|
app.UseCors();
|
||||||
|
|
||||||
|
app.UseMiddleware<GatewayReadinessMiddleware>();
|
||||||
|
|
||||||
app.MapReverseProxy().RequireRateLimiting("fixed");
|
app.MapReverseProxy().RequireRateLimiting("fixed");
|
||||||
|
|
||||||
app.MapControllers();
|
app.MapControllers();
|
||||||
|
|
||||||
app.Run();
|
app.Run();
|
||||||
Reference in New Issue
Block a user