diff --git a/src/ServiceControl.Federation.UnitTests/.editorconfig b/src/ServiceControl.Federation.UnitTests/.editorconfig
new file mode 100644
index 0000000000..b9691251b6
--- /dev/null
+++ b/src/ServiceControl.Federation.UnitTests/.editorconfig
@@ -0,0 +1,7 @@
+[*.cs]
+
+# Justification: Application synchronization contexts don't require ConfigureAwait(false)
+dotnet_diagnostic.CA2007.severity = none
+
+# Justification: test helpers that start in-process hosts do not need cancellation
+dotnet_diagnostic.PS0018.severity = none
diff --git a/src/ServiceControl.Federation.UnitTests/FederationEndToEndTests.cs b/src/ServiceControl.Federation.UnitTests/FederationEndToEndTests.cs
new file mode 100644
index 0000000000..6fe0f0c93d
--- /dev/null
+++ b/src/ServiceControl.Federation.UnitTests/FederationEndToEndTests.cs
@@ -0,0 +1,349 @@
+namespace ServiceControl.Federation.UnitTests;
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using System.Text.Json.Nodes;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Hosting.Server;
+using Microsoft.AspNetCore.Hosting.Server.Features;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Routing.Template;
+using Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using NUnit.Framework;
+
+///
+/// Runs the proxy against two fake members over real HTTP and checks what ServicePulse would see.
+///
+[TestFixture]
+[NonParallelizable]
+class FederationEndToEndTests
+{
+ [SetUp]
+ public async Task SetUp()
+ {
+ memberA = await FakeMember.Start("a");
+ memberB = await FakeMember.Start("b");
+ (proxy, client) = await StartProxy(memberA, memberB);
+ }
+
+ [TearDown]
+ public async Task TearDown()
+ {
+ client.Dispose();
+ await proxy.DisposeAsync();
+ await memberA.DisposeAsync();
+ await memberB.DisposeAsync();
+ }
+
+ [Test]
+ public async Task Failed_message_pages_are_merged_across_members()
+ {
+ memberA.Respond("GET", "/api/errors", ctx => List(ctx, Failures("a", 0, 2, 4)));
+ memberB.Respond("GET", "/api/errors", ctx => List(ctx, Failures("b", 1, 3, 5)));
+
+ var response = await client.GetAsync("api/errors?status=unresolved&page=2&per_page=2&sort=time_of_failure&direction=asc");
+ var rows = JsonNode.Parse(await response.Content.ReadAsStringAsync())!.AsArray();
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(response.StatusCode, Is.EqualTo(HttpStatusCode.OK));
+ Assert.That(rows.Select(r => r!["id"]!.GetValue()), Is.EqualTo(new[] { "a-2", "b-3" }));
+ Assert.That(response.Headers.GetValues("Total-Count").Single(), Is.EqualTo("6"));
+ Assert.That(memberA.Requests, Has.Some.Contains("page=1").And.Contains("per_page=4"), "each member is asked for its first page·per_page rows");
+ }
+ }
+
+ [Test]
+ public async Task Batch_commands_reach_only_the_owning_member()
+ {
+ memberA.Respond("GET", "/api/errors", ctx => List(ctx, Failures("a", 0)));
+ memberB.Respond("GET", "/api/errors", ctx => List(ctx, Failures("b", 1)));
+ memberA.Respond("POST", "/api/errors/retry", _ => Results.Accepted());
+ memberB.Respond("POST", "/api/errors/retry", _ => Results.Accepted());
+
+ await client.GetAsync("api/errors");
+ var response = await client.PostAsync("api/errors/retry", Json("[\"b-1\"]"));
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(response.StatusCode, Is.EqualTo(HttpStatusCode.Accepted));
+ Assert.That(memberB.Bodies("POST /api/errors/retry"), Is.EqualTo(new[] { "[\"b-1\"]" }));
+ Assert.That(memberA.Bodies("POST /api/errors/retry"), Is.Empty);
+ }
+ }
+
+ [Test]
+ public async Task Unknown_ids_are_found_by_probing_members()
+ {
+ memberA.Respond("GET", "/api/errors/last/{id}", _ => Results.NotFound());
+ memberB.Respond("GET", "/api/errors/last/{id}", _ => Results.Json(new { id = "b-9" }));
+ memberB.Respond("PATCH", "/api/errors/archive", _ => Results.Accepted());
+
+ var response = await client.PatchAsync("api/errors/archive", Json("[\"b-9\"]"));
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(response.StatusCode, Is.EqualTo(HttpStatusCode.Accepted));
+ Assert.That(memberB.Bodies("PATCH /api/errors/archive"), Has.Count.EqualTo(1));
+ Assert.That(memberA.Requests, Has.None.StartsWith("PATCH"));
+ }
+ }
+
+ [Test]
+ public async Task Root_document_points_at_the_proxy_and_separates_display_and_gating_versions()
+ {
+ memberA.Version = "6.12.0";
+ memberB.Version = "6.10.1";
+ memberA.Respond("GET", "/api", ctx => Results.Json(new { name = "A", my_routes_url = $"{memberA.BaseUrl}api/my/routes", license_status = "valid" }));
+ memberB.Respond("GET", "/api", ctx => Results.Json(new { name = "B", my_routes_url = $"{memberB.BaseUrl}api/my/routes", license_status = "valid" }));
+
+ var response = await client.GetAsync("api");
+ var body = JsonNode.Parse(await response.Content.ReadAsStringAsync())!;
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(response.Headers.GetValues("X-Particular-Version").Single(), Is.EqualTo("6.10.1"), "feature gates read the lowest member version");
+ Assert.That(body["platform_health_version"]!.GetValue(), Is.EqualTo("6.12.0"), "platform health shows the lead member's own version");
+ Assert.That(body["my_routes_url"]!.GetValue(), Is.EqualTo($"{client.BaseAddress}api/my/routes"));
+ Assert.That(body["name"]!.GetValue(), Is.EqualTo("Federation under test"));
+ }
+ }
+
+ [Test]
+ public async Task Instance_names_are_qualified_so_internal_checks_match_their_own_installation()
+ {
+ static IResult Audit() => Results.Json(new[] { new { api_uri = "http://servicecontrol-audit:44444", status = "online", configuration = new { host = new { instance_name = "Particular.ServiceControl.Audit" } } } });
+ memberA.Respond("GET", "/api/configuration/remotes", _ => Audit());
+ memberB.Respond("GET", "/api/configuration/remotes", _ => Audit());
+ memberB.Respond("GET", "/api/configuration", _ => Results.Json(new { host = new { instance_name = "Particular.ServiceControl" } }));
+ memberA.Respond("GET", "/api/customchecks", ctx => List(ctx, [Check("c1", internalCheck: true), Check("c2", internalCheck: false)]));
+ memberB.Respond("GET", "/api/customchecks", ctx => List(ctx, []));
+
+ var remotes = JsonNode.Parse(await client.GetStringAsync("api/configuration/remotes"))!.AsArray();
+ var checks = JsonNode.Parse(await client.GetStringAsync("api/customchecks?status=fail"))!.AsArray();
+
+ string Name(JsonNode? node, string path) => Aggregation.JsonValues.Text(node, path)!;
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(remotes.Select(r => Name(r, "configuration.host.instance_name")), Is.EqualTo(new[]
+ {
+ "Particular.ServiceControl.Audit (a)",
+ "Particular.ServiceControl.Audit (b)",
+ "Particular.ServiceControl (b)"
+ }), "audit instances with the same address in two installations stay separate");
+ Assert.That(checks.Select(c => Name(c, "originating_endpoint.name")), Is.EqualTo(new[] { "Particular.ServiceControl.Audit (a)", "Sales" }), "only internal checks are renamed");
+ }
+ }
+
+ static JsonNode Check(string id, bool internalCheck) => new JsonObject
+ {
+ ["id"] = id,
+ ["internal"] = internalCheck,
+ ["reported_at"] = DateTime.UnixEpoch.ToString("O"),
+ ["originating_endpoint"] = new JsonObject { ["name"] = internalCheck ? "Particular.ServiceControl.Audit" : "Sales" }
+ };
+
+ [Test]
+ public async Task Monitoring_root_shows_the_lead_version_and_the_health_of_every_member()
+ {
+ memberA.Version = "6.12.0";
+ memberB.Version = "6.10.1";
+ memberA.Respond("GET", "/", _ => Results.Json(new { instanceType = "monitoring", version = "6.12.0" }));
+ memberB.Respond("GET", "/", _ => Results.Json(new { instanceType = "monitoring", version = "6.10.1" }));
+
+ var healthy = await client.GetAsync("monitoring");
+ var healthyBody = JsonNode.Parse(await healthy.Content.ReadAsStringAsync())!;
+
+ await memberB.DisposeAsync();
+ var degradedBody = JsonNode.Parse(await client.GetStringAsync("monitoring"))!;
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(healthy.Headers.GetValues("X-Particular-Version").Single(), Is.EqualTo("6.10.1"));
+ Assert.That(healthyBody["platform_health_version"]!.GetValue(), Is.EqualTo("6.12.0"));
+ Assert.That(healthyBody["platform_health_status"]!.GetValue(), Is.EqualTo("healthy"));
+ Assert.That(degradedBody["platform_health_status"]!.GetValue(), Is.EqualTo("degraded"), "one Monitoring row stands for every member");
+ }
+ }
+
+ [Test]
+ public async Task Message_bodies_are_routed_by_instance_id()
+ {
+ memberA.Respond("GET", "/api/messages", ctx => List(ctx, [new JsonObject { ["id"] = "m", ["message_id"] = "m", ["body_url"] = "/messages/m/body?instance_id=audit1" }]));
+ memberB.Respond("GET", "/api/messages", ctx => List(ctx, []));
+ memberA.Respond("GET", "/api/messages/{id}/body", ctx => Results.Text($"body from {ctx.Request.QueryString}", "text/plain"));
+
+ var list = JsonNode.Parse(await client.GetStringAsync("api/messages"))!.AsArray();
+ var bodyUrl = list[0]!["body_url"]!.GetValue();
+ var body = await client.GetStringAsync("api" + bodyUrl);
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(bodyUrl, Is.EqualTo("/messages/m/body?instance_id=a~audit1"));
+ Assert.That(body, Is.EqualTo("body from ?instance_id=audit1"));
+ }
+ }
+
+ [Test]
+ public async Task Failure_groups_are_merged_by_id()
+ {
+ memberA.Respond("GET", "/api/recoverability/groups/{classifier}", _ => Results.Json(new[] { new { id = "g", title = "T", count = 2, last = "2024-01-01T00:00:00Z", operation_status = "None" } }));
+ memberB.Respond("GET", "/api/recoverability/groups/{classifier}", _ => Results.Json(new[] { new { id = "g", title = "T", count = 3, last = "2024-02-01T00:00:00Z", operation_status = "None" } }));
+
+ var groups = JsonNode.Parse(await client.GetStringAsync("api/recoverability/groups/Exception%20Type%20and%20Stack%20Trace"))!.AsArray();
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(groups, Has.Count.EqualTo(1));
+ Assert.That(groups[0]!["count"]!.GetValue(), Is.EqualTo(5));
+ Assert.That(memberA.Requests, Has.Some.Contains("Exception%20Type%20and%20Stack%20Trace"), "the path is forwarded as the client encoded it");
+ }
+ }
+
+ [Test]
+ public async Task An_unreachable_member_is_reported_as_incomplete()
+ {
+ memberA.Respond("GET", "/api/customchecks", ctx => List(ctx, []));
+ await memberB.DisposeAsync();
+
+ var response = await client.GetAsync("api/customchecks?status=fail");
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(response.StatusCode, Is.EqualTo(HttpStatusCode.OK));
+ Assert.That(response.Headers.GetValues("X-Particular-Incomplete-Results").Single(), Does.EndWith(":unavailable"));
+ }
+ }
+
+ static IResult List(HttpContext context, IEnumerable rows)
+ {
+ var all = rows.ToList();
+ var page = int.TryParse(context.Request.Query["page"], out var p) ? p : 1;
+ var perPage = int.TryParse(context.Request.Query["per_page"], out var pp) ? pp : 50;
+ context.Response.Headers["Total-Count"] = all.Count.ToString(System.Globalization.CultureInfo.InvariantCulture);
+ return Results.Content(new JsonArray([.. all.Skip((page - 1) * perPage).Take(perPage)]).ToJsonString(), "application/json");
+ }
+
+ static IEnumerable Failures(string member, params int[] minutes) =>
+ minutes.Select(minute => (JsonNode)new JsonObject
+ {
+ ["id"] = $"{member}-{minute}",
+ ["message_id"] = $"{member}-{minute}",
+ ["time_of_failure"] = DateTime.UnixEpoch.AddMinutes(minute).ToString("O")
+ });
+
+ static StringContent Json(string json) => new(json, Encoding.UTF8, "application/json");
+
+ static async Task<(WebApplication, HttpClient)> StartProxy(params FakeMember[] members)
+ {
+ // The tests address the federated surfaces at /api and /monitoring for readability; a hosting
+ // primary uses FederationSettings.DefaultPrimaryPathBase and DefaultMonitoringPathBase.
+ var settings = new FederationSettings
+ {
+ DisplayName = "Federation under test",
+ MemberTimeout = TimeSpan.FromSeconds(5),
+ HealthProbeInterval = TimeSpan.FromHours(1),
+ PrimaryPathBase = "/api",
+ MonitoringPathBase = "/monitoring",
+ Members = [.. members.Select(m => new MemberSettings { Key = m.Key, PrimaryUrl = $"{m.BaseUrl}api/", MonitoringUrl = m.BaseUrl })]
+ };
+
+ var builder = WebApplication.CreateSlimBuilder();
+ builder.Logging.ClearProviders();
+ builder.WebHost.UseUrls("http://127.0.0.1:0");
+ builder.Services.AddServiceControlFederation(settings);
+
+ var app = builder.Build();
+ app.UseRouting();
+ app.MapServiceControlFederation();
+ await app.StartAsync();
+
+ var client = new HttpClient { BaseAddress = new Uri(Address(app)) };
+ return (app, client);
+ }
+
+ static string Address(WebApplication app) =>
+ app.Services.GetRequiredService().Features.Get()!.Addresses.First().TrimEnd('/') + "/";
+
+ FakeMember memberA = null!;
+ FakeMember memberB = null!;
+ WebApplication proxy = null!;
+ HttpClient client = null!;
+
+ sealed class FakeMember : IAsyncDisposable
+ {
+ FakeMember(string key) => Key = key;
+
+ public string Key { get; }
+
+ public string BaseUrl { get; private set; } = "";
+
+ public string Version { get; set; } = "6.12.0";
+
+ public ConcurrentQueue Requests { get; } = new();
+
+ public List Bodies(string methodAndPath) =>
+ [.. bodies.Where(b => b.Key == methodAndPath).Select(b => b.Value)];
+
+ public void Respond(string method, string path, Func handler) =>
+ handlers.Enqueue((method, new TemplateMatcher(TemplateParser.Parse(path.TrimStart('/')), []), handler));
+
+ public static async Task Start(string key)
+ {
+ var member = new FakeMember(key);
+ var builder = WebApplication.CreateSlimBuilder();
+ builder.Logging.ClearProviders();
+ builder.WebHost.UseUrls("http://127.0.0.1:0");
+ var app = builder.Build();
+ app.Run(async context =>
+ {
+ member.Requests.Enqueue($"{context.Request.Method} {context.Request.Path}{context.Request.QueryString}");
+ if (context.Request.ContentLength > 0)
+ {
+ using var reader = new System.IO.StreamReader(context.Request.Body);
+ member.bodies.Add(new KeyValuePair($"{context.Request.Method} {context.Request.Path}", await reader.ReadToEndAsync()));
+ }
+
+ context.Response.Headers["X-Particular-Version"] = member.Version;
+ foreach (var (method, matcher, handler) in member.handlers)
+ {
+ if (method == context.Request.Method && matcher.TryMatch(context.Request.Path, context.Request.RouteValues))
+ {
+ await handler(context).ExecuteAsync(context);
+ return;
+ }
+ }
+
+ context.Response.StatusCode = StatusCodes.Status404NotFound;
+ });
+ member.app = app;
+ await app.StartAsync();
+ member.BaseUrl = Address(app);
+ return member;
+ }
+
+ public async ValueTask DisposeAsync()
+ {
+ if (app != null)
+ {
+ var stopping = app;
+ app = null;
+ await stopping.DisposeAsync();
+ }
+ }
+
+ readonly ConcurrentBag> bodies = [];
+ readonly ConcurrentQueue<(string Method, TemplateMatcher Matcher, Func Handler)> handlers = new();
+ WebApplication? app;
+ }
+}
diff --git a/src/ServiceControl.Federation.UnitTests/FederationSettingsTests.cs b/src/ServiceControl.Federation.UnitTests/FederationSettingsTests.cs
new file mode 100644
index 0000000000..f7710abd77
--- /dev/null
+++ b/src/ServiceControl.Federation.UnitTests/FederationSettingsTests.cs
@@ -0,0 +1,58 @@
+namespace ServiceControl.Federation.UnitTests;
+
+using System;
+using System.Linq;
+using Configuration;
+using NUnit.Framework;
+
+[TestFixture]
+class FederationSettingsTests
+{
+ [Test]
+ public void Instances_are_parsed_in_the_remote_instances_style()
+ {
+ var members = FederationSettings.ParseMembers("""
+ [
+ { "api_uri": "http://sc-eu:33333/api/", "monitoring_uri": "http://mon-eu:33633/", "key": "eu", "name": "Europe" },
+ { "api_uri": "http://sc-us:33333/api/" }
+ ]
+ """);
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(members.Select(m => m.Key), Is.EqualTo(new[] { "eu", "sc-us-33333" }), "a missing key is derived from the api_uri host and port");
+ Assert.That(members[0].Name, Is.EqualTo("Europe"));
+ Assert.That(members[0].MonitoringUrl, Is.EqualTo("http://mon-eu:33633/"));
+ Assert.That(members[1].MonitoringUrl, Is.Null);
+ }
+ }
+
+ [TestCase(null)]
+ [TestCase("")]
+ [TestCase("[]")]
+ public void No_instances_fail_validation(string? json)
+ {
+ var settings = new FederationSettings { Members = [.. FederationSettings.ParseMembers(json)] };
+
+ Assert.That(settings.Validate, Throws.InvalidOperationException.With.Message.Contains("FederatedInstances"));
+ }
+
+ [Test]
+ public void Instances_that_derive_the_same_key_must_be_given_distinct_keys()
+ {
+ var settings = new FederationSettings
+ {
+ Members = [.. FederationSettings.ParseMembers("""[{ "api_uri": "http://localhost:33333/api/" }, { "api_uri": "http://localhost:33333/api" }]""")]
+ };
+
+ Assert.That(settings.Validate, Throws.InvalidOperationException.With.Message.Contains("distinct"));
+ }
+
+ [Test]
+ public void Invalid_urls_fail_validation()
+ {
+ var settings = new FederationSettings { Members = [new MemberSettings { Key = "a", PrimaryUrl = "not a url" }] };
+
+ Assert.That(settings.Validate, Throws.InstanceOf().With.Message.Contains("api_uri"));
+ }
+}
diff --git a/src/ServiceControl.Federation.UnitTests/MergerTests.cs b/src/ServiceControl.Federation.UnitTests/MergerTests.cs
new file mode 100644
index 0000000000..a2dbb4c85e
--- /dev/null
+++ b/src/ServiceControl.Federation.UnitTests/MergerTests.cs
@@ -0,0 +1,163 @@
+namespace ServiceControl.Federation.UnitTests;
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json.Nodes;
+using Aggregation;
+using NUnit.Framework;
+using Routing;
+
+[TestFixture]
+class FailureGroupMergerTests
+{
+ [Test]
+ public void Groups_with_the_same_id_are_combined()
+ {
+ var merged = FailureGroupMerger.MergeLists(
+ [
+ [Group("g1", 3, "2024-01-01T00:00:00Z", "2024-01-05T00:00:00Z")],
+ [Group("g1", 4, "2023-12-01T00:00:00Z", "2024-01-02T00:00:00Z"), Group("g2", 1, "2024-02-01T00:00:00Z", "2024-02-01T00:00:00Z")]
+ ]);
+
+ Assert.That(merged, Has.Count.EqualTo(2));
+ var g1 = merged.Single(g => JsonValues.Text(g, "id") == "g1");
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(JsonValues.Number(g1, "count"), Is.EqualTo(7));
+ Assert.That(JsonValues.Text(g1, "first"), Is.EqualTo("2023-12-01T00:00:00Z"));
+ Assert.That(JsonValues.Text(g1, "last"), Is.EqualTo("2024-01-05T00:00:00Z"));
+ Assert.That(JsonValues.Text(merged[0], "id"), Is.EqualTo("g2"), "ordered by last, newest first");
+ }
+ }
+
+ [Test]
+ public void Operation_state_reflects_the_most_active_member()
+ {
+ var done = Group("g1", 0, null, null);
+ done["operation_status"] = "Completed";
+ done["operation_progress"] = 1.0;
+ done["operation_remaining_count"] = 0;
+ done["operation_messages_completed_count"] = 30;
+ done["operation_completion_time"] = "2024-01-01T00:00:00Z";
+
+ var running = Group("g1", 10, null, null);
+ running["operation_status"] = "Forwarding";
+ running["operation_progress"] = 0.0;
+ running["operation_remaining_count"] = 10;
+ running["operation_messages_completed_count"] = 0;
+
+ var idle = Group("g1", 5, null, null);
+ idle["operation_status"] = "None";
+
+ var merged = FailureGroupMerger.Merge([done, running, idle]);
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(JsonValues.Text(merged, "operation_status"), Is.EqualTo("Forwarding"));
+ Assert.That(JsonValues.Number(merged, "operation_progress"), Is.EqualTo(0.75));
+ Assert.That(JsonValues.Number(merged, "operation_remaining_count"), Is.EqualTo(10));
+ Assert.That(merged["operation_completion_time"], Is.Null, "not complete until every member is");
+ }
+ }
+
+ static JsonObject Group(string id, int count, string? first, string? last) => new()
+ {
+ ["id"] = id,
+ ["title"] = "System.Exception",
+ ["type"] = "Exception Type and Stack Trace",
+ ["count"] = count,
+ ["first"] = first,
+ ["last"] = last,
+ ["operation_status"] = "None"
+ };
+}
+
+[TestFixture]
+class MonitoredEndpointMergerTests
+{
+ [Test]
+ public void Endpoints_with_the_same_name_are_combined()
+ {
+ var merged = MonitoredEndpointMerger.MergeList(
+ [
+ [Endpoint("Sales", stale: false, throughput: [10, 10], processingTime: [100, 100])],
+ [Endpoint("Sales", stale: true, throughput: [30, 0], processingTime: [200, 0]), Endpoint("Billing", stale: true, throughput: [1, 1], processingTime: [1, 1])]
+ ]);
+
+ Assert.That(merged, Has.Count.EqualTo(2));
+ var sales = merged[0]!;
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(JsonValues.Bool(sales, "isStale"), Is.False);
+ Assert.That(Points(sales, "throughput"), Is.EqualTo(new[] { 40.0, 10.0 }));
+ Assert.That(Points(sales, "processingTime"), Is.EqualTo(new[] { 175.0, 100.0 }), "weighted by throughput");
+ Assert.That((JsonValues.At(sales, "endpointInstanceIds") as JsonArray)!, Has.Count.EqualTo(2));
+ }
+ }
+
+ static IEnumerable Points(JsonNode endpoint, string metric) =>
+ (JsonValues.At(endpoint, $"metrics.{metric}.points") as JsonArray)!.Select(JsonValues.AsNumber);
+
+ static JsonObject Endpoint(string name, bool stale, double[] throughput, double[] processingTime) => new()
+ {
+ ["name"] = name,
+ ["isStale"] = stale,
+ ["endpointInstanceIds"] = new JsonArray($"{name}-{stale}"),
+ ["metrics"] = new JsonObject
+ {
+ ["throughput"] = new JsonObject { ["average"] = throughput.Average(), ["points"] = new JsonArray([.. throughput.Select(p => (JsonNode)p)]) },
+ ["processingTime"] = new JsonObject { ["average"] = processingTime.Average(), ["points"] = new JsonArray([.. processingTime.Select(p => (JsonNode)p)]) }
+ }
+ };
+}
+
+[TestFixture]
+class InstanceIdCodecTests
+{
+ [TestCase("/messages/1/body?instance_id=aHR0cDovL2E.", "a", "/messages/1/body?instance_id=a~aHR0cDovL2E.")]
+ [TestCase("/messages/1/body", "a", "/messages/1/body?instance_id=a~")]
+ [TestCase("/messages/1/body?x=1", "b", "/messages/1/body?x=1&instance_id=b~")]
+ public void Body_urls_carry_the_member_key(string url, string member, string expected) =>
+ Assert.That(InstanceIdCodec.RewriteBodyUrl(url, member), Is.EqualTo(expected));
+
+ [Test]
+ public void Encoded_ids_round_trip()
+ {
+ Assert.That(InstanceIdCodec.TryDecode(InstanceIdCodec.Encode("eu", "aHR0cDovL2E."), out var key, out var id), Is.True);
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(key, Is.EqualTo("eu"));
+ Assert.That(id, Is.EqualTo("aHR0cDovL2E."));
+ }
+
+ Assert.That(InstanceIdCodec.TryDecode("aHR0cDovL2E.", out _, out _), Is.False, "a member's own id is not routable");
+ }
+
+ [Test]
+ public void Trees_are_rewritten_in_place()
+ {
+ var row = new JsonObject
+ {
+ ["instance_id"] = "abc",
+ ["body_url"] = "/messages/1/body?instance_id=abc",
+ ["nested"] = new JsonArray(new JsonObject { ["body_url"] = "/messages/2/body" })
+ };
+
+ InstanceIdCodec.RewriteTree(row, "eu");
+
+ using (Assert.EnterMultipleScope())
+ {
+ Assert.That(JsonValues.Text(row, "instance_id"), Is.EqualTo("eu~abc"));
+ Assert.That(JsonValues.Text(row, "body_url"), Is.EqualTo("/messages/1/body?instance_id=eu~abc"));
+ Assert.That(JsonValues.Text(row["nested"]![0], "body_url"), Is.EqualTo("/messages/2/body?instance_id=eu~"));
+ }
+ }
+}
+
+[TestFixture]
+class VersionsTests
+{
+ [Test]
+ public void The_lowest_version_wins() =>
+ Assert.That(Versions.Lowest(["6.10.0", "6.9.1", null, "6.9.1-alpha.3+sha"]), Is.EqualTo("6.9.1"));
+}
diff --git a/src/ServiceControl.Federation.UnitTests/NUnitSettings.cs b/src/ServiceControl.Federation.UnitTests/NUnitSettings.cs
new file mode 100644
index 0000000000..8b76b23ac7
--- /dev/null
+++ b/src/ServiceControl.Federation.UnitTests/NUnitSettings.cs
@@ -0,0 +1,5 @@
+using NUnit.Framework;
+
+[assembly: FixtureLifeCycle(LifeCycle.InstancePerTestCase)]
+[assembly: Parallelizable(ParallelScope.All)]
+[assembly: LevelOfParallelism(4)]
diff --git a/src/ServiceControl.Federation.UnitTests/RouteCoverageTests.cs b/src/ServiceControl.Federation.UnitTests/RouteCoverageTests.cs
new file mode 100644
index 0000000000..4b4acd0249
--- /dev/null
+++ b/src/ServiceControl.Federation.UnitTests/RouteCoverageTests.cs
@@ -0,0 +1,91 @@
+namespace ServiceControl.Federation.UnitTests;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Api;
+using Members;
+using NUnit.Framework;
+
+///
+/// Every route a ServiceControl instance exposes must have a federation strategy. The approved route
+/// lists are the instances' own approval files, so adding a route to ServiceControl fails this test
+/// until the route is classified in or .
+///
+[TestFixture]
+class RouteCoverageTests
+{
+ [Test]
+ public void Every_primary_route_has_a_strategy()
+ {
+ var approved = ApprovedRoutes("Primary.txt", PrimaryPrefix)
+ .Concat(ApprovedRoutes("Licensing.txt", _ => "licensing/"));
+
+ AssertCovered(approved, PrimaryRoutes.All);
+ }
+
+ [Test]
+ public void Every_monitoring_route_has_a_strategy() =>
+ AssertCovered(ApprovedRoutes("Monitoring.txt", MonitoringPrefix), MonitoringRoutes.All);
+
+ [Test]
+ public void Route_tables_have_no_duplicates()
+ {
+ var duplicates = FederationHostingExtensions.AllRoutes
+ .SelectMany(route => route.Methods.Select(method => $"{route.Surface} {method} {Normalise(route.Template)}"))
+ .GroupBy(key => key)
+ .Where(group => group.Count() > 1)
+ .Select(group => group.Key)
+ .ToList();
+
+ Assert.That(duplicates, Is.Empty);
+ }
+
+ static void AssertCovered(IEnumerable approved, IEnumerable routes)
+ {
+ var proxied = routes
+ .SelectMany(route => route.Methods.Select(method => $"{method} {Normalise(route.Template)}"))
+ .ToHashSet(StringComparer.OrdinalIgnoreCase);
+
+ var missing = approved.Where(route => !proxied.Contains(route)).ToList();
+
+ Assert.That(missing, Is.Empty, "These routes have no federation strategy. Add them to the route table.");
+ }
+
+ // The approval files print only the method-level template; these controllers add a prefix.
+ static string PrimaryPrefix(string controller) => controller switch
+ {
+ _ when controller.EndsWith("AuthenticationController", StringComparison.Ordinal) => "authentication/",
+ _ when controller.EndsWith("SystemMaintenanceController", StringComparison.Ordinal) => "maintenance/",
+ _ => ""
+ };
+
+ static string MonitoringPrefix(string controller) =>
+ controller.EndsWith("MyRoutesController", StringComparison.Ordinal) ? "api/" : "";
+
+ static IEnumerable ApprovedRoutes(string file, Func prefixFor)
+ {
+ var path = Path.Combine(TestContext.CurrentContext.TestDirectory, "ApprovedRoutes", file);
+ foreach (var line in File.ReadAllLines(path).Where(l => l.Contains("=>", StringComparison.Ordinal)))
+ {
+ var match = RouteLine().Match(line);
+ Assert.That(match.Success, Is.True, $"Unexpected approval line: {line}");
+
+ var controller = match.Groups["controller"].Value;
+ var template = prefixFor(controller) + match.Groups["template"].Value.TrimStart('/');
+ foreach (var method in match.Groups["methods"].Value.Split('/'))
+ {
+ yield return $"{method} {Normalise(template)}";
+ }
+ }
+ }
+
+ static string Normalise(string template) =>
+ string.Join('/', template.Trim('/').Split('/').Select(segment => Parameter().Replace(segment, "{}"))).ToLowerInvariant();
+
+ static Regex RouteLine() => new(@"^(?[A-Z/]+) (?\S+) => (?[^:]+):");
+
+ static Regex Parameter() => new(@"\{[^}]*\}");
+}
diff --git a/src/ServiceControl.Federation.UnitTests/ServiceControl.Federation.UnitTests.csproj b/src/ServiceControl.Federation.UnitTests/ServiceControl.Federation.UnitTests.csproj
new file mode 100644
index 0000000000..e35b4862ad
--- /dev/null
+++ b/src/ServiceControl.Federation.UnitTests/ServiceControl.Federation.UnitTests.csproj
@@ -0,0 +1,33 @@
+
+
+
+ net10.0
+ enable
+ DefaultCore
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/ServiceControl.Federation.UnitTests/SortedPageMergerTests.cs b/src/ServiceControl.Federation.UnitTests/SortedPageMergerTests.cs
new file mode 100644
index 0000000000..9cf4f6ea72
--- /dev/null
+++ b/src/ServiceControl.Federation.UnitTests/SortedPageMergerTests.cs
@@ -0,0 +1,77 @@
+namespace ServiceControl.Federation.UnitTests;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json.Nodes;
+using Aggregation;
+using NUnit.Framework;
+
+[TestFixture]
+class SortedPageMergerTests
+{
+ [Test]
+ public void Merged_pages_match_a_global_sort([Values(1, 2, 3, 7)] int page, [Values(5, 10)] int pageSize, [Range(0, 9)] int seed)
+ {
+ var random = new Random(seed);
+ var members = Enumerable.Range(0, random.Next(1, 5))
+ .Select(member => Enumerable.Range(0, random.Next(0, 40))
+ .Select(index => Row($"{member}-{index}", DateTime.UnixEpoch.AddMinutes(random.Next(0, 500))))
+ .ToList())
+ .ToList();
+
+ var order = RowOrder.FailedMessages("time_of_failure", "desc");
+ var depth = page * pageSize;
+
+ // What each member returns for page=1&per_page=depth
+ var memberPages = members
+ .Select(rows => (IReadOnlyList)[.. rows.OrderBy(r => r, order).Take(depth)])
+ .ToList();
+
+ var merged = SortedPageMerger.Merge(memberPages, order, depth - pageSize, pageSize);
+ var expected = members.SelectMany(rows => rows).OrderBy(r => r, order).Skip(depth - pageSize).Take(pageSize);
+
+ Assert.That(merged.Select(Id), Is.EqualTo(expected.Select(Id)));
+ }
+
+ [Test]
+ public void Duplicates_keep_the_earliest_member()
+ {
+ var lead = Row("lead", DateTime.UnixEpoch, endpoint: "Sales", messageId: "m1");
+ var other = Row("other", DateTime.UnixEpoch.AddMinutes(1), endpoint: "Sales", messageId: "m1");
+
+ var merged = SortedPageMerger.Merge(
+ [[lead], [other]],
+ RowOrder.Messages("time_sent", "desc"),
+ 0,
+ 10,
+ row => $"{JsonValues.Text(row, "receiving_endpoint.name")}-{JsonValues.Text(row, "message_id")}");
+
+ Assert.That(merged.Select(Id), Is.EqualTo(new[] { "lead" }));
+ }
+
+ [Test]
+ public void Durations_sort_by_value_not_text()
+ {
+ JsonNode Timed(string id, string criticalTime) => new JsonObject { ["id"] = id, ["critical_time"] = criticalTime };
+
+ var merged = SortedPageMerger.Merge(
+ [[Timed("long", "1.00:00:00")], [Timed("short", "09:00:00")]],
+ RowOrder.Messages("critical_time", "asc"),
+ 0,
+ 10);
+
+ Assert.That(merged.Select(Id), Is.EqualTo(new[] { "short", "long" }));
+ }
+
+ static JsonNode Row(string id, DateTime time, string endpoint = "Endpoint", string? messageId = null) => new JsonObject
+ {
+ ["id"] = id,
+ ["message_id"] = messageId ?? id,
+ ["time_of_failure"] = time.ToString("O"),
+ ["time_sent"] = time.ToString("O"),
+ ["receiving_endpoint"] = new JsonObject { ["name"] = endpoint }
+ };
+
+ static string? Id(JsonNode row) => JsonValues.Text(row, "id");
+}
diff --git a/src/ServiceControl.Federation/.editorconfig b/src/ServiceControl.Federation/.editorconfig
new file mode 100644
index 0000000000..aff82c0034
--- /dev/null
+++ b/src/ServiceControl.Federation/.editorconfig
@@ -0,0 +1,4 @@
+[*.cs]
+
+# Justification: Application synchronization contexts don't require ConfigureAwait(false)
+dotnet_diagnostic.CA2007.severity = none
diff --git a/src/ServiceControl.Federation/Aggregation/CommandOutcome.cs b/src/ServiceControl.Federation/Aggregation/CommandOutcome.cs
new file mode 100644
index 0000000000..c2d6e6a262
--- /dev/null
+++ b/src/ServiceControl.Federation/Aggregation/CommandOutcome.cs
@@ -0,0 +1,47 @@
+namespace ServiceControl.Federation.Aggregation;
+
+using System.Collections.Generic;
+using System.Linq;
+using Members;
+
+///
+/// Picks the response a broadcast or partitioned command relays to the caller.
+///
+public static class CommandOutcome
+{
+ ///
+ /// The member response to relay, or null when no member answered at all.
+ ///
+ ///
+ ///
+ /// - Any success, with the rest 404 or failed: the success. A 404 means the member does not hold
+ /// the resource, which is expected when a command reaches every member.
+ /// - Any 4xx other than 404 and no success: that client error, so its reason reaches the caller.
+ /// - All 404: a 404.
+ /// - Otherwise: the first failure.
+ ///
+ /// Among successes, 202 wins over 200 because the command is still in progress somewhere.
+ ///
+ public static MemberResponse? Choose(IReadOnlyList responses)
+ {
+ var successes = responses.Where(r => r.IsSuccess).ToList();
+ if (successes.Count > 0)
+ {
+ return successes.FirstOrDefault(r => r.Status == 202) ?? successes[0];
+ }
+
+ var clientError = responses.FirstOrDefault(r => r.Failure == null && r.Status is >= 400 and < 500 and not 404);
+ if (clientError != null)
+ {
+ return clientError;
+ }
+
+ var notFound = responses.FirstOrDefault(r => r.IsNotFound);
+ if (notFound != null && responses.All(r => r.IsNotFound))
+ {
+ return notFound;
+ }
+
+ return responses.FirstOrDefault(r => r.Failure != null) ?? notFound;
+ }
+}
diff --git a/src/ServiceControl.Federation/Aggregation/FailureGroupMerger.cs b/src/ServiceControl.Federation/Aggregation/FailureGroupMerger.cs
new file mode 100644
index 0000000000..39885a4404
--- /dev/null
+++ b/src/ServiceControl.Federation/Aggregation/FailureGroupMerger.cs
@@ -0,0 +1,180 @@
+namespace ServiceControl.Federation.Aggregation;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json.Nodes;
+
+///
+/// Merges failure groups (GroupOperation rows) by id. A group id is
+/// DeterministicGuid.MakeId(classifier, title), so the same group has the same id on every
+/// member and the merged row describes the group across all of them.
+///
+public static class FailureGroupMerger
+{
+ ///
+ /// The maximum number of groups a member returns, kept for the merged list too.
+ ///
+ public const int MaxGroups = 200;
+
+ /// One list per member, lead member first.
+ public static JsonArray MergeLists(IReadOnlyList> memberGroups)
+ {
+ var byId = new Dictionary>(StringComparer.OrdinalIgnoreCase);
+ var order = new List();
+
+ foreach (var groups in memberGroups)
+ {
+ foreach (var group in groups)
+ {
+ var id = JsonValues.Text(group, "id");
+ if (id == null)
+ {
+ continue;
+ }
+
+ if (!byId.TryGetValue(id, out var instances))
+ {
+ instances = [];
+ byId[id] = instances;
+ order.Add(id);
+ }
+
+ instances.Add(group);
+ }
+ }
+
+ var merged = order
+ .Select(id => Merge(byId[id]))
+ .OrderByDescending(group => JsonValues.TimeTicks(group, "last") ?? long.MinValue)
+ .Take(MaxGroups);
+
+ return [.. merged];
+ }
+
+ ///
+ /// Combines the views several members hold of one group.
+ ///
+ public static JsonObject Merge(IReadOnlyList instances)
+ {
+ var result = instances[0].DeepClone().AsObject();
+ if (instances.Count == 1)
+ {
+ return result;
+ }
+
+ result["count"] = instances.Sum(i => (long)(JsonValues.Number(i, "count") ?? 0));
+ SetTimeExtreme(result, instances, "first", min: true);
+ SetTimeExtreme(result, instances, "last", min: false);
+ result["comment"] = MergeComment(instances);
+ MergeOperation(result, instances);
+
+ return result;
+ }
+
+ static string? MergeComment(IReadOnlyList instances)
+ {
+ var comments = instances
+ .Select(i => JsonValues.Text(i, "comment"))
+ .Where(c => !string.IsNullOrEmpty(c))
+ .ToList();
+
+ // Differing comments can only come from edits made directly against a member; the proxy
+ // broadcasts comment edits. The lead member's comment wins because it is listed first.
+ return comments.FirstOrDefault();
+ }
+
+ static void MergeOperation(JsonObject result, IReadOnlyList instances)
+ {
+ var participating = instances.Where(i => StatusRank(JsonValues.Text(i, "operation_status")) > 0).ToList();
+ if (participating.Count == 0)
+ {
+ return;
+ }
+
+ var status = participating
+ .Select(i => JsonValues.Text(i, "operation_status"))
+ .OrderByDescending(StatusRank)
+ .First();
+ result["operation_status"] = status;
+
+ var allComplete = participating.All(i => StatusRank(JsonValues.Text(i, "operation_status")) == CompletedRank);
+
+ var weights = participating.Select(Weight).ToList();
+ var totalWeight = weights.Sum();
+ var progress = totalWeight == 0
+ ? participating.Average(i => JsonValues.Number(i, "operation_progress") ?? 0)
+ : participating.Select((i, index) => (JsonValues.Number(i, "operation_progress") ?? 0) * weights[index]).Sum() / totalWeight;
+ result["operation_progress"] = progress;
+
+ result["operation_remaining_count"] = SumOrNull(participating, "operation_remaining_count");
+ result["operation_messages_completed_count"] = SumOrNull(participating, "operation_messages_completed_count");
+
+ var failed = participating.Select(i => JsonValues.Bool(i, "operation_failed")).ToList();
+ result["operation_failed"] = failed.All(f => f == null) ? null : failed.Any(f => f == true);
+
+ result["need_user_acknowledgement"] = instances.Any(i => JsonValues.Bool(i, "need_user_acknowledgement") == true);
+
+ SetTimeExtreme(result, participating, "operation_start_time", min: true);
+ if (allComplete)
+ {
+ SetTimeExtreme(result, participating, "operation_completion_time", min: false);
+ }
+ else
+ {
+ result["operation_completion_time"] = null;
+ }
+
+ SetTimeExtreme(result, instances, "last_operation_completion_time", min: false);
+ }
+
+ static double Weight(JsonNode group)
+ {
+ var remaining = JsonValues.Number(group, "operation_remaining_count");
+ var completed = JsonValues.Number(group, "operation_messages_completed_count");
+ if (remaining != null || completed != null)
+ {
+ return (remaining ?? 0) + (completed ?? 0);
+ }
+
+ return JsonValues.Number(group, "count") ?? 0;
+ }
+
+ static long? SumOrNull(IReadOnlyList instances, string field)
+ {
+ var values = instances.Select(i => JsonValues.Number(i, field)).Where(v => v != null).ToList();
+ return values.Count == 0 ? null : values.Sum(v => (long)v!.Value);
+ }
+
+ static void SetTimeExtreme(JsonObject result, IReadOnlyList instances, string field, bool min)
+ {
+ var candidates = instances
+ .Select(i => (Node: JsonValues.At(i, field), Ticks: JsonValues.TimeTicks(i, field)))
+ .Where(c => c.Ticks != null)
+ .ToList();
+
+ if (candidates.Count == 0)
+ {
+ return;
+ }
+
+ var winner = min ? candidates.MinBy(c => c.Ticks) : candidates.MaxBy(c => c.Ticks);
+ result[field] = winner.Node?.DeepClone();
+ }
+
+ ///
+ /// How active an operation state is. The merged state is the most active one any member reports,
+ /// so a group shows as in progress until every member has finished.
+ ///
+ public static int StatusRank(string? status) => status switch
+ {
+ null or "" or "None" => 0,
+ "Completed" or "ArchiveCompleted" => CompletedRank,
+ "Waiting" => 3,
+ "Forwarding" or "ArchiveProgressing" or "ArchiveFinalizing" => 4,
+ "Preparing" or "ArchiveStarted" => 5,
+ _ => 3
+ };
+
+ const int CompletedRank = 1;
+}
diff --git a/src/ServiceControl.Federation/Aggregation/JsonValues.cs b/src/ServiceControl.Federation/Aggregation/JsonValues.cs
new file mode 100644
index 0000000000..80fdd97128
--- /dev/null
+++ b/src/ServiceControl.Federation/Aggregation/JsonValues.cs
@@ -0,0 +1,79 @@
+namespace ServiceControl.Federation.Aggregation;
+
+using System;
+using System.Globalization;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+///
+/// Tolerant readers for the handful of fields the proxy inspects. Member payloads are handled as JSON
+/// trees so that fields the proxy does not know about pass through unchanged.
+///
+static class JsonValues
+{
+ ///
+ /// Reads a dotted path such as receiving_endpoint.name.
+ ///
+ public static JsonNode? At(JsonNode? node, string path)
+ {
+ var current = node;
+ foreach (var segment in path.Split('.'))
+ {
+ if (current is not JsonObject obj || !obj.TryGetPropertyValue(segment, out current))
+ {
+ return null;
+ }
+ }
+
+ return current;
+ }
+
+ public static string? Text(JsonNode? node, string path) => AsText(At(node, path));
+
+ public static string? AsText(JsonNode? node) =>
+ node is JsonValue value && value.GetValueKind() is not JsonValueKind.Null and not JsonValueKind.Object and not JsonValueKind.Array
+ ? value.GetValueKind() == JsonValueKind.String ? value.GetValue() : value.ToJsonString()
+ : null;
+
+ public static double? Number(JsonNode? node, string path) => AsNumber(At(node, path));
+
+ public static double? AsNumber(JsonNode? node)
+ {
+ if (node is not JsonValue value)
+ {
+ return null;
+ }
+
+ // Parsed values hold a JsonElement, but values created in code hold the CLR type they were
+ // created from, which GetValue does not convert. The JSON text works for both.
+ var kind = value.GetValueKind();
+ string? text = null;
+ if (kind == JsonValueKind.Number)
+ {
+ text = value.ToJsonString();
+ }
+ else if (kind == JsonValueKind.String)
+ {
+ text = value.GetValue();
+ }
+
+ return double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) ? parsed : null;
+ }
+
+ public static bool? Bool(JsonNode? node, string path) =>
+ At(node, path) is JsonValue value && value.GetValueKind() is JsonValueKind.True or JsonValueKind.False
+ ? value.GetValue()
+ : null;
+
+ public static long? TimeTicks(JsonNode? node, string path) => AsTimeTicks(At(node, path));
+
+ public static long? AsTimeTicks(JsonNode? node) =>
+ AsText(node) is { } text && DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var time)
+ ? time.UtcTicks
+ : null;
+
+ public static long? DurationTicks(JsonNode? node, string path) =>
+ Text(node, path) is { } text && TimeSpan.TryParse(text, CultureInfo.InvariantCulture, out var duration)
+ ? duration.Ticks
+ : null;
+}
diff --git a/src/ServiceControl.Federation/Aggregation/MonitoredEndpointMerger.cs b/src/ServiceControl.Federation/Aggregation/MonitoredEndpointMerger.cs
new file mode 100644
index 0000000000..ce6a86949f
--- /dev/null
+++ b/src/ServiceControl.Federation/Aggregation/MonitoredEndpointMerger.cs
@@ -0,0 +1,225 @@
+namespace ServiceControl.Federation.Aggregation;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json.Nodes;
+
+///
+/// Merges Monitoring responses. Endpoints are keyed by name only, so an endpoint reported by more than
+/// one member is combined: counts and rates add up, timings are averaged weighted by throughput.
+/// Every member uses the same history period, and each series has the same number of buckets aligned
+/// to wall-clock UTC, so points combine index by index.
+///
+public static class MonitoredEndpointMerger
+{
+ static readonly HashSet SummedMetrics = new(StringComparer.OrdinalIgnoreCase) { "throughput", "retries", "queueLength" };
+
+ /// One monitored-endpoints list per member, lead member first.
+ public static JsonArray MergeList(IReadOnlyList> memberEndpoints)
+ {
+ var byName = new Dictionary>(StringComparer.Ordinal);
+ var order = new List();
+
+ foreach (var endpoints in memberEndpoints)
+ {
+ foreach (var endpoint in endpoints)
+ {
+ var name = JsonValues.Text(endpoint, "name") ?? "";
+ if (!byName.TryGetValue(name, out var instances))
+ {
+ instances = [];
+ byName[name] = instances;
+ order.Add(name);
+ }
+
+ instances.Add(endpoint);
+ }
+ }
+
+ return [.. order.Select(name => MergeEndpoint(byName[name]))];
+ }
+
+ public static JsonObject MergeEndpoint(IReadOnlyList instances)
+ {
+ var result = instances[0].DeepClone().AsObject();
+ if (instances.Count == 1)
+ {
+ return result;
+ }
+
+ result["isStale"] = instances.All(i => JsonValues.Bool(i, "isStale") ?? false);
+ result["connectedCount"] = instances.Sum(i => (long)(JsonValues.Number(i, "connectedCount") ?? 0));
+ result["disconnectedCount"] = instances.Sum(i => (long)(JsonValues.Number(i, "disconnectedCount") ?? 0));
+ result["endpointInstanceIds"] = new JsonArray([.. instances
+ .SelectMany(i => JsonValues.At(i, "endpointInstanceIds") as JsonArray ?? [])
+ .Select(id => id?.DeepClone())]);
+ result["metrics"] = MergeMetrics([.. instances.Select(i => JsonValues.At(i, "metrics") as JsonObject)]);
+
+ return result;
+ }
+
+ ///
+ /// Merges monitored-endpoints/{name} details. Members that do not know the endpoint answer
+ /// with an empty document, so only members reporting instances take part.
+ ///
+ public static JsonObject MergeDetails(IReadOnlyList memberDetails)
+ {
+ var reporting = memberDetails
+ .Where(d => (JsonValues.At(d, "instances") as JsonArray)?.Count > 0)
+ .ToList();
+
+ if (reporting.Count == 0)
+ {
+ return memberDetails.Count > 0 ? memberDetails[0].DeepClone().AsObject() : [];
+ }
+
+ var result = reporting[0].DeepClone().AsObject();
+ if (reporting.Count == 1)
+ {
+ return result;
+ }
+
+ result["instances"] = new JsonArray([.. reporting
+ .SelectMany(d => JsonValues.At(d, "instances") as JsonArray ?? [])
+ .Select(i => i?.DeepClone())]);
+
+ var messageTypes = new JsonArray();
+ var seenTypes = new HashSet(StringComparer.Ordinal);
+ foreach (var type in reporting.SelectMany(d => JsonValues.At(d, "messageTypes") as JsonArray ?? []))
+ {
+ if (type != null && seenTypes.Add(JsonValues.Text(type, "id") ?? type.ToJsonString()))
+ {
+ messageTypes.Add(type.DeepClone());
+ }
+ }
+
+ result["messageTypes"] = messageTypes;
+
+ if (result["digest"] is JsonObject digest)
+ {
+ digest["metrics"] = MergeMetrics([.. reporting.Select(d => JsonValues.At(d, "digest.metrics") as JsonObject)]);
+ }
+
+ if (result["metricDetails"] is JsonObject metricDetails)
+ {
+ metricDetails["metrics"] = MergeMetrics([.. reporting.Select(d => JsonValues.At(d, "metricDetails.metrics") as JsonObject)]);
+ }
+
+ return result;
+ }
+
+ ///
+ /// Merges metric maps ({ name: { average, latest, points[], timeAxisValues[] } }), one per member.
+ ///
+ public static JsonObject MergeMetrics(IReadOnlyList memberMetrics)
+ {
+ var present = memberMetrics.Where(m => m != null).Select(m => m!).ToList();
+ if (present.Count == 0)
+ {
+ return [];
+ }
+
+ if (present.Count == 1)
+ {
+ return present[0].DeepClone().AsObject();
+ }
+
+ var result = new JsonObject();
+ var names = present.SelectMany(m => m.Select(p => p.Key)).Distinct(StringComparer.Ordinal).ToList();
+ var throughput = present.Select(m => m.TryGetPropertyValue("throughput", out var t) ? t : null).ToList();
+
+ foreach (var name in names)
+ {
+ var values = present.Select(m => m.TryGetPropertyValue(name, out var v) ? v : null).ToList();
+ var template = values.First(v => v != null)!.DeepClone().AsObject();
+ var summed = SummedMetrics.Contains(name);
+
+ foreach (var scalar in new[] { "average", "latest" })
+ {
+ if (template.ContainsKey(scalar))
+ {
+ template[scalar] = CombineScalar(values, throughput, scalar, summed);
+ }
+ }
+
+ if (template.ContainsKey("points"))
+ {
+ template["points"] = CombinePoints(values, throughput, summed);
+ }
+
+ result[name] = template;
+ }
+
+ return result;
+ }
+
+ static double? CombineScalar(IReadOnlyList values, IReadOnlyList throughput, string field, bool summed)
+ {
+ var samples = values
+ .Select((v, i) => (Value: JsonValues.Number(v, field), Weight: JsonValues.Number(throughput[i], field) ?? 0))
+ .Where(s => s.Value != null)
+ .ToList();
+
+ if (samples.Count == 0)
+ {
+ return null;
+ }
+
+ if (summed)
+ {
+ return samples.Sum(s => s.Value!.Value);
+ }
+
+ var totalWeight = samples.Sum(s => s.Weight);
+ return totalWeight > 0
+ ? samples.Sum(s => s.Value!.Value * s.Weight) / totalWeight
+ : samples.Average(s => s.Value!.Value);
+ }
+
+ static JsonArray CombinePoints(IReadOnlyList values, IReadOnlyList throughput, bool summed)
+ {
+ var series = values.Select(v => JsonValues.At(v, "points") as JsonArray).ToList();
+ var weights = throughput.Select(t => JsonValues.At(t, "points") as JsonArray).ToList();
+ var length = series.Max(s => s?.Count ?? 0);
+ var result = new JsonArray();
+
+ for (var index = 0; index < length; index++)
+ {
+ var total = 0.0;
+ var weightedTotal = 0.0;
+ var totalWeight = 0.0;
+ var samples = 0;
+
+ for (var member = 0; member < series.Count; member++)
+ {
+ var point = series[member] is { } s && index < s.Count ? JsonValues.AsNumber(s[index]) : null;
+ if (point == null)
+ {
+ continue;
+ }
+
+ var weight = weights[member] is { } w && index < w.Count ? JsonValues.AsNumber(w[index]) ?? 0 : 0;
+ total += point.Value;
+ weightedTotal += point.Value * weight;
+ totalWeight += weight;
+ samples++;
+ }
+
+ if (samples == 0)
+ {
+ result.Add(0);
+ }
+ else if (summed)
+ {
+ result.Add(total);
+ }
+ else
+ {
+ result.Add(totalWeight > 0 ? weightedTotal / totalWeight : total / samples);
+ }
+ }
+
+ return result;
+ }
+}
diff --git a/src/ServiceControl.Federation/Aggregation/RowOrder.cs b/src/ServiceControl.Federation/Aggregation/RowOrder.cs
new file mode 100644
index 0000000000..087d92d903
--- /dev/null
+++ b/src/ServiceControl.Federation/Aggregation/RowOrder.cs
@@ -0,0 +1,115 @@
+namespace ServiceControl.Federation.Aggregation;
+
+using System;
+using System.Collections.Generic;
+using System.Text.Json.Nodes;
+
+public enum ValueKind
+{
+ Text,
+ Time,
+ Duration,
+ Number
+}
+
+///
+/// Reproduces a member's sort order over JSON rows so that rows from several members can be merged
+/// into one correctly ordered list. Ties fall back to an ordinal comparison of a tie-breaker field;
+/// the merge itself is stable, so rows that still tie keep member order (lead first).
+///
+public sealed class RowOrder(string field, ValueKind kind, bool descending, string? tieBreaker = null) : IComparer
+{
+ public string Field { get; } = field;
+
+ public bool Descending { get; } = descending;
+
+ public int Compare(JsonNode? x, JsonNode? y)
+ {
+ var result = CompareField(x, y, Field, kind);
+ if (Descending)
+ {
+ result = -result;
+ }
+
+ if (result == 0 && tieBreaker != null)
+ {
+ result = string.CompareOrdinal(JsonValues.Text(x, tieBreaker), JsonValues.Text(y, tieBreaker));
+ }
+
+ return result;
+ }
+
+ static int CompareField(JsonNode? x, JsonNode? y, string path, ValueKind kind) => kind switch
+ {
+ ValueKind.Time => Nullable.Compare(JsonValues.TimeTicks(x, path), JsonValues.TimeTicks(y, path)),
+ ValueKind.Duration => Nullable.Compare(JsonValues.DurationTicks(x, path), JsonValues.DurationTicks(y, path)),
+ ValueKind.Number => Nullable.Compare(JsonValues.Number(x, path), JsonValues.Number(y, path)),
+ ValueKind.Text => string.CompareOrdinal(JsonValues.Text(x, path), JsonValues.Text(y, path)),
+ _ => 0
+ };
+
+ static bool IsDescending(string? direction, bool defaultDescending) => direction switch
+ {
+ "asc" => false,
+ "desc" => true,
+ _ => defaultDescending
+ };
+
+ ///
+ /// Failed message lists (errors, endpoints/{name}/errors, group errors). Mirrors
+ /// FailedMessageQueryFilters.Sort: unknown sort tokens fall back to time_sent.
+ ///
+ public static RowOrder FailedMessages(string? sort, string? direction)
+ {
+ var descending = IsDescending(direction, true);
+ return sort switch
+ {
+ "id" or "message_id" => new RowOrder("message_id", ValueKind.Text, descending, "id"),
+ "message_type" => new RowOrder("message_type", ValueKind.Text, descending, "id"),
+ "status" => new RowOrder("status", ValueKind.Text, descending, "id"),
+ "modified" => new RowOrder("last_modified", ValueKind.Time, descending, "id"),
+ "time_of_failure" => new RowOrder("time_of_failure", ValueKind.Time, descending, "id"),
+ _ => new RowOrder("time_sent", ValueKind.Time, descending, "id")
+ };
+ }
+
+ ///
+ /// Audit and composite message lists. Mirrors MessageViewComparer.
+ ///
+ public static RowOrder Messages(string? sort, string? direction)
+ {
+ var descending = IsDescending(direction, true);
+ return sort switch
+ {
+ "id" or "message_id" => new RowOrder("message_id", ValueKind.Text, descending, "id"),
+ "message_type" => new RowOrder("message_type", ValueKind.Text, descending, "id"),
+ "status" => new RowOrder("status", ValueKind.Text, descending, "id"),
+ "processed_at" => new RowOrder("processed_at", ValueKind.Time, descending, "id"),
+ "critical_time" => new RowOrder("critical_time", ValueKind.Duration, descending, "id"),
+ "delivery_time" => new RowOrder("delivery_time", ValueKind.Duration, descending, "id"),
+ "processing_time" => new RowOrder("processing_time", ValueKind.Duration, descending, "id"),
+ _ => new RowOrder("time_sent", ValueKind.Time, descending, "id")
+ };
+ }
+
+ ///
+ /// Custom checks are ordered by report time, oldest first, and take no sort parameters.
+ ///
+ public static RowOrder CustomChecks() => new("reported_at", ValueKind.Time, false, "id");
+
+ ///
+ /// The event log is ordered newest first and takes no sort parameters.
+ ///
+ public static RowOrder EventLog() => new("raised_at", ValueKind.Time, true, "id");
+
+ ///
+ /// Mirrors MessageRedirectsSortingExtensions.Sort.
+ ///
+ public static RowOrder Redirects(string? sort, string? direction)
+ {
+ var descending = IsDescending(direction, true);
+ return string.Equals(sort, "to_physical_address", StringComparison.OrdinalIgnoreCase)
+ ? new RowOrder("to_physical_address", ValueKind.Text, descending, "message_redirect_id")
+ : new RowOrder("from_physical_address", ValueKind.Text, descending, "message_redirect_id");
+ }
+}
diff --git a/src/ServiceControl.Federation/Aggregation/SortedPageMerger.cs b/src/ServiceControl.Federation/Aggregation/SortedPageMerger.cs
new file mode 100644
index 0000000000..d2687b5e1e
--- /dev/null
+++ b/src/ServiceControl.Federation/Aggregation/SortedPageMerger.cs
@@ -0,0 +1,51 @@
+namespace ServiceControl.Federation.Aggregation;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json.Nodes;
+
+///
+/// Merges sorted row lists from several members into one page.
+///
+///
+/// Page p of size s holds rows (p-1)·s … p·s-1 of the global order, and no member
+/// can contribute more than its own first p·s rows to them. Asking each member for its first
+/// p·s rows and merging is therefore exact. ServiceControl's own scatter-gather instead asks every
+/// instance for its page p, which is only exact for page 1.
+///
+public static class SortedPageMerger
+{
+ /// One list per member, each already sorted by , lead member first.
+ /// The order the members sorted by.
+ /// Rows of the merged order to skip.
+ /// Rows to return.
+ /// Rows with an equal key are kept once; the earliest member wins.
+ public static List Merge(
+ IReadOnlyList> memberRows,
+ IComparer order,
+ int skip,
+ int take,
+ Func? deduplicationKey = null)
+ {
+ var seen = deduplicationKey == null ? null : new HashSet(StringComparer.Ordinal);
+ var candidates = new List();
+
+ foreach (var rows in memberRows)
+ {
+ foreach (var row in rows)
+ {
+ var key = deduplicationKey?.Invoke(row);
+ if (seen != null && key != null && !seen.Add(key))
+ {
+ continue;
+ }
+
+ candidates.Add(row);
+ }
+ }
+
+ // OrderBy is stable, so rows that compare equal keep member order.
+ return [.. candidates.OrderBy(row => row, order).Skip(skip).Take(take)];
+ }
+}
diff --git a/src/ServiceControl.Federation/Aggregation/Versions.cs b/src/ServiceControl.Federation/Aggregation/Versions.cs
new file mode 100644
index 0000000000..435a4c40d5
--- /dev/null
+++ b/src/ServiceControl.Federation/Aggregation/Versions.cs
@@ -0,0 +1,35 @@
+namespace ServiceControl.Federation.Aggregation;
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+
+///
+/// ServicePulse gates features on X-Particular-Version. The proxy reports the lowest member
+/// version so it never enables a feature that one member lacks.
+///
+public static class Versions
+{
+ public static string? Lowest(IEnumerable versions) =>
+ versions
+ .Where(v => !string.IsNullOrWhiteSpace(v))
+ .Select(v => v!)
+ .OrderBy(Parse)
+ .ThenBy(v => v, StringComparer.Ordinal)
+ .FirstOrDefault();
+
+ ///
+ /// Reads the numeric major.minor.patch prefix, ignoring pre-release and build metadata.
+ ///
+ public static Version Parse(string version)
+ {
+ var numeric = version.Split('-', '+')[0];
+ var parts = numeric.Split('.')
+ .Select(p => int.TryParse(p, NumberStyles.Integer, CultureInfo.InvariantCulture, out var n) ? n : 0)
+ .Concat([0, 0, 0])
+ .Take(3)
+ .ToArray();
+ return new Version(parts[0], parts[1], parts[2]);
+ }
+}
diff --git a/src/ServiceControl.Federation/Api/.editorconfig b/src/ServiceControl.Federation/Api/.editorconfig
new file mode 100644
index 0000000000..98abd8819c
--- /dev/null
+++ b/src/ServiceControl.Federation/Api/.editorconfig
@@ -0,0 +1,4 @@
+[*.cs]
+
+# Justification: route handlers receive HttpContext, whose RequestAborted is the request's cancellation token
+dotnet_diagnostic.PS0018.severity = none
diff --git a/src/ServiceControl.Federation/Api/FederatedRoute.cs b/src/ServiceControl.Federation/Api/FederatedRoute.cs
new file mode 100644
index 0000000000..0e995bbd70
--- /dev/null
+++ b/src/ServiceControl.Federation/Api/FederatedRoute.cs
@@ -0,0 +1,36 @@
+namespace ServiceControl.Federation.Api;
+
+using System.Threading.Tasks;
+using Members;
+using Microsoft.AspNetCore.Http;
+
+public delegate Task RouteHandler(Federator federator, HttpContext context);
+
+///
+/// Strategy labels used in the route tables, matching the design document.
+///
+public static class Strategy
+{
+ public const string Union = "U";
+ public const string Sum = "S";
+ public const string KeyMerge = "K";
+ public const string SortedPage = "P";
+ public const string TopN = "W";
+ public const string FirstHit = "F";
+ public const string RouteToOwner = "R";
+ public const string Broadcast = "B";
+ public const string Lead = "L";
+ public const string Synthesised = "X";
+}
+
+///
+/// One proxied route: the methods and template it answers, relative to the surface prefix, and how.
+///
+public sealed record FederatedRoute(Surface Surface, string[] Methods, string Template, string Strategy, RouteHandler Handler)
+{
+ ///
+ /// Routes the members serve without authentication, such as the root document and the
+ /// authentication configuration a client needs before it can sign in.
+ ///
+ public bool Anonymous { get; init; }
+}
diff --git a/src/ServiceControl.Federation/Api/Federator.cs b/src/ServiceControl.Federation/Api/Federator.cs
new file mode 100644
index 0000000000..1696a3ff0d
--- /dev/null
+++ b/src/ServiceControl.Federation/Api/Federator.cs
@@ -0,0 +1,450 @@
+namespace ServiceControl.Federation.Api;
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Threading.Tasks;
+using Aggregation;
+using Configuration;
+using Http;
+using Members;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+using Routing;
+
+///
+/// The reusable request strategies. Each proxied route is one of these, sometimes with a
+/// route-specific merge on top (see and ).
+///
+public sealed class Federator(
+ MemberRegistry registry,
+ MemberGateway gateway,
+ OwnerCache owners,
+ OwnerResolver resolver,
+ FederationSettings settings,
+ ILogger logger)
+{
+ public MemberRegistry Registry => registry;
+
+ public MemberGateway Gateway => gateway;
+
+ public OwnerCache Owners => owners;
+
+ public FederationSettings Settings => settings;
+
+ public ILogger Logger => logger;
+
+ public string Prefix(Surface surface) => surface == Surface.Primary ? settings.PrimaryPathBase : settings.MonitoringPathBase;
+
+ public IReadOnlyList Members(Surface surface) => registry.Serving(surface);
+
+ public string? LowestVersion(Surface surface) => Versions.Lowest(Members(surface).Select(m => m.Version(surface)));
+
+ public string Target(HttpContext context, Surface surface) => ProxyRequest.RelativeTarget(context, Prefix(surface));
+
+ public Task FanOut(HttpContext context, Surface surface, string target) =>
+ gateway.SendAll(Members(surface), surface, ProxyRequest.Get(context, target), context.RequestAborted);
+
+ public Task FanOut(HttpContext context, Surface surface) => FanOut(context, surface, Target(context, surface));
+
+ public ResponseMeta Meta(Surface surface, IReadOnlyCollection responses, long? totalCount = null, string variant = "", string? link = null, IEnumerable? extraIncomplete = null) =>
+ new()
+ {
+ TotalCount = totalCount,
+ ETag = EntityTags.Combine(responses, variant),
+ Version = LowestVersion(surface),
+ Link = link,
+ Incomplete = [.. ProxyResponse.IncompleteEntries(responses, surface), .. extraIncomplete ?? []]
+ };
+
+ ///
+ /// Ends the request when no member produced a usable answer. A 401 or 403 is relayed as is, so the
+ /// client can renew its token or explain the refusal.
+ ///
+ public async Task NothingAnswered(HttpContext context, Surface surface, IReadOnlyList responses, bool notFoundIsAnAnswer = true)
+ {
+ if (responses.Any(r => r.IsSuccess) || (notFoundIsAnAnswer && responses.Any(r => r.IsNotFound)))
+ {
+ return false;
+ }
+
+ var authFailure = responses.FirstOrDefault(r => r.IsAuthFailure);
+ if (authFailure != null)
+ {
+ await ProxyResponse.Relay(context, authFailure, context.RequestAborted);
+ return true;
+ }
+
+ var clientError = responses.FirstOrDefault(r => r.Failure == null && r.Status is >= 400 and < 500 and not 404);
+ if (clientError != null)
+ {
+ await ProxyResponse.Relay(context, clientError, context.RequestAborted);
+ return true;
+ }
+
+ var incomplete = ProxyResponse.IncompleteEntries(responses, surface);
+ if (responses.Count == 0)
+ {
+ await ProxyResponse.Problem(context, StatusCodes.Status404NotFound, $"No member serves the {surface} API.", incomplete, context.RequestAborted);
+ }
+ else if (responses.Any(r => r.Failure == FailureReason.Timeout))
+ {
+ await ProxyResponse.Problem(context, StatusCodes.Status504GatewayTimeout, "No member answered in time.", incomplete, context.RequestAborted);
+ }
+ else
+ {
+ await ProxyResponse.Problem(context, StatusCodes.Status502BadGateway, "No member answered.", incomplete, context.RequestAborted);
+ }
+
+ return true;
+ }
+
+ ///
+ /// L: answered by the lead member, or by the next member when the lead is down.
+ ///
+ public async Task Lead(HttpContext context, Surface surface)
+ {
+ var request = await ProxyRequest.Repeat(context, Prefix(surface), context.RequestAborted);
+ var failures = new List();
+ foreach (var member in Members(surface))
+ {
+ var response = await gateway.Send(member, surface, request, context.RequestAborted);
+ if (response.Failure == null)
+ {
+ await ProxyResponse.Relay(context, response, context.RequestAborted);
+ return;
+ }
+
+ failures.Add(response);
+ }
+
+ await NothingAnswered(context, surface, failures);
+ }
+
+ ///
+ /// L, streamed: for large or binary payloads such as license uploads.
+ ///
+ public async Task LeadStreaming(HttpContext context, Surface surface)
+ {
+ var request = await ProxyRequest.Repeat(context, Prefix(surface), context.RequestAborted);
+ foreach (var member in Members(surface))
+ {
+ using var response = await gateway.SendStreaming(member, surface, request, context.RequestAborted);
+ if (response != null)
+ {
+ await ProxyResponse.RelayStreaming(context, response, context.RequestAborted);
+ return;
+ }
+ }
+
+ await ProxyResponse.Problem(context, StatusCodes.Status502BadGateway, "No member answered.", cancellationToken: context.RequestAborted);
+ }
+
+ ///
+ /// B: the command goes to every member (or the given ones); see .
+ ///
+ public async Task Broadcast(HttpContext context, Surface surface, IReadOnlyList? targets = null)
+ {
+ var request = await ProxyRequest.Repeat(context, Prefix(surface), context.RequestAborted);
+ var responses = await gateway.SendAll(targets ?? Members(surface), surface, request, context.RequestAborted);
+ await RelayOutcome(context, surface, responses);
+ }
+
+ public async Task RelayOutcome(HttpContext context, Surface surface, IReadOnlyList responses)
+ {
+ var chosen = CommandOutcome.Choose(responses);
+ if (chosen == null || chosen.Status == 0)
+ {
+ await NothingAnswered(context, surface, responses, notFoundIsAnAnswer: false);
+ return;
+ }
+
+ var incomplete = ProxyResponse.IncompleteEntries(responses.Where(r => r != chosen), surface);
+ if (incomplete.Count > 0)
+ {
+ context.Response.Headers[ProxyResponse.IncompleteResultsHeader] = string.Join(", ", incomplete);
+ }
+
+ await ProxyResponse.Relay(context, chosen, context.RequestAborted);
+ }
+
+ ///
+ /// F: the first member holding the resource answers. Members are asked in parallel; the earliest
+ /// member in registry order wins, so the lead member is preferred.
+ ///
+ public async Task FirstHit(HttpContext context, Surface surface, OwnedRecord? kind = null, bool rewriteInstanceIds = false)
+ {
+ var responses = await FanOut(context, surface);
+ if (await NothingAnswered(context, surface, responses))
+ {
+ return;
+ }
+
+ var hits = responses.Where(r => r.IsSuccess).ToList();
+ if (hits.Count == 0)
+ {
+ await ProxyResponse.Problem(context, StatusCodes.Status404NotFound, "Not found on any member.", ProxyResponse.IncompleteEntries(responses, surface), context.RequestAborted);
+ return;
+ }
+
+ foreach (var hit in hits)
+ {
+ if (kind is { } recordKind)
+ {
+ owners.Record(recordKind, JsonValues.Text(hit.Json(), "id"), hit.Member.Key);
+ }
+ }
+
+ var body = hits[0].Json();
+ if (rewriteInstanceIds)
+ {
+ InstanceIdCodec.RewriteTree(body, hits[0].Member.Key);
+ }
+
+ await ProxyResponse.WriteJson(context, body, Meta(surface, responses), context.RequestAborted);
+ }
+
+ ///
+ /// S over Total-Count: for HEAD count endpoints.
+ ///
+ public async Task SumCount(HttpContext context, Surface surface)
+ {
+ var request = await ProxyRequest.Repeat(context, Prefix(surface), context.RequestAborted);
+ var responses = await gateway.SendAll(Members(surface), surface, request, context.RequestAborted);
+ if (await NothingAnswered(context, surface, responses))
+ {
+ return;
+ }
+
+ var total = responses.Where(r => r.IsSuccess).Sum(r => (long)(r.TotalCount ?? 0));
+ await ProxyResponse.WriteJson(context, null, Meta(surface, responses, total), context.RequestAborted);
+ }
+
+ ///
+ /// U: member arrays are concatenated, optionally de-duplicated by key (earliest member wins).
+ ///
+ public async Task Union(HttpContext context, Surface surface, Func? deduplicationKey = null, OwnedRecord? kind = null, string idField = "id", bool totalFromRows = false)
+ {
+ var responses = await FanOut(context, surface);
+ if (await NothingAnswered(context, surface, responses))
+ {
+ return;
+ }
+
+ var seen = new HashSet(StringComparer.Ordinal);
+ var result = new JsonArray();
+ foreach (var response in responses.Where(r => r.IsSuccess))
+ {
+ foreach (var row in response.Array().OfType())
+ {
+ if (kind is { } recordKind)
+ {
+ owners.Record(recordKind, JsonValues.Text(row, idField), response.Member.Key);
+ }
+
+ var key = deduplicationKey?.Invoke(row);
+ if (key != null && !seen.Add(key))
+ {
+ continue;
+ }
+
+ result.Add(row.DeepClone());
+ }
+ }
+
+ long? total = totalFromRows
+ ? result.Count
+ : responses.Any(r => r.TotalCount != null) ? responses.Sum(r => (long)(r.TotalCount ?? 0)) : null;
+
+ await ProxyResponse.WriteJson(context, result, Meta(surface, responses, total), context.RequestAborted);
+ }
+
+ ///
+ /// P: exact merge of sorted, paged lists (see ).
+ ///
+ public async Task SortedPage(
+ HttpContext context,
+ Surface surface,
+ RowOrder order,
+ OwnedRecord? kind = null,
+ string idField = "id",
+ Func? deduplicationKey = null,
+ Action? transform = null)
+ {
+ var page = ProxyRequest.IntQuery(context, "page", 1);
+ var pageSize = ProxyRequest.IntQuery(context, "per_page", DefaultPageSize);
+ var depth = (long)page * pageSize;
+ var exact = depth <= settings.DeepPagingMaxRows;
+
+ var target = exact
+ ? ProxyRequest.WithQuery(context, Prefix(surface), ProxyRequest.QueryWith(context, ("page", "1"), ("per_page", depth.ToString(CultureInfo.InvariantCulture))))
+ : Target(context, surface);
+
+ var responses = await FanOut(context, surface, target);
+ if (await NothingAnswered(context, surface, responses))
+ {
+ return;
+ }
+
+ var memberRows = responses
+ .Select(response => (IReadOnlyList)[.. RowsOf(response, kind, idField, transform)])
+ .ToList();
+
+ var merged = SortedPageMerger.Merge(memberRows, order, exact ? (int)(depth - pageSize) : 0, pageSize, deduplicationKey);
+ var total = responses.Where(r => r.IsSuccess).Sum(r => (long)(r.TotalCount ?? 0));
+ var link = ProxyResponse.PagingLinks(context, Prefix(surface), page, pageSize, total);
+ string[] approximate = exact ? [] : [$"{ProxyRequest.BaseUrl(context, Prefix(surface)).TrimEnd('/')}:approximate-paging"];
+
+ await ProxyResponse.WriteJson(
+ context,
+ new JsonArray([.. merged.Select(row => row.DeepClone())]),
+ Meta(surface, responses, total, target, link, approximate),
+ context.RequestAborted);
+ }
+
+ ///
+ /// W: lists without offsets (messages2) are forwarded unchanged; the merged top N is exact.
+ ///
+ public async Task TopN(HttpContext context, Surface surface, RowOrder order, string pageSizeParameter, Func? deduplicationKey = null, Action? transform = null)
+ {
+ var responses = await FanOut(context, surface);
+ if (await NothingAnswered(context, surface, responses))
+ {
+ return;
+ }
+
+ var take = ProxyRequest.IntQuery(context, pageSizeParameter, int.MaxValue);
+ var memberRows = responses
+ .Select(response => (IReadOnlyList)[.. RowsOf(response, null, "id", transform)])
+ .ToList();
+
+ var merged = SortedPageMerger.Merge(memberRows, order, 0, take, deduplicationKey);
+ var total = responses.Where(r => r.IsSuccess).Sum(r => (long)(r.TotalCount ?? 0));
+
+ await ProxyResponse.WriteJson(context, new JsonArray([.. merged.Select(row => row.DeepClone())]), Meta(surface, responses, total), context.RequestAborted);
+ }
+
+ IEnumerable RowsOf(MemberResponse response, OwnedRecord? kind, string idField, Action? transform)
+ {
+ if (!response.IsSuccess)
+ {
+ yield break;
+ }
+
+ foreach (var row in response.Array().OfType())
+ {
+ transform?.Invoke(row, response.Member);
+ if (kind is { } recordKind)
+ {
+ owners.Record(recordKind, JsonValues.Text(row, idField), response.Member.Key);
+ }
+
+ yield return row;
+ }
+ }
+
+ ///
+ /// R: the command goes only to the members holding the record.
+ ///
+ public async Task RouteToOwners(HttpContext context, Surface surface, OwnedRecord kind, string id)
+ {
+ var request = await ProxyRequest.Repeat(context, Prefix(surface), context.RequestAborted);
+ var resolution = await resolver.Resolve(context, kind, [id], context.RequestAborted);
+ if (resolution.Owners.Count == 0)
+ {
+ logger.LogWarning("No member holds {Kind} {Id}; {Method} {Path} was not forwarded", kind, id, context.Request.Method, context.Request.Path);
+ await ProxyResponse.Problem(context, StatusCodes.Status404NotFound, $"No member holds {kind} '{id}'.", cancellationToken: context.RequestAborted);
+ return;
+ }
+
+ var responses = await gateway.SendAll(resolution.Owners.Keys, surface, request, context.RequestAborted);
+ await RelayOutcome(context, surface, responses);
+ }
+
+ ///
+ /// R, partitioned: a body listing ids (an array, or an object with uniquemessageids) is split
+ /// so each owner receives only its ids. Any other body (a time range, for example) is broadcast.
+ ///
+ public async Task Partitioned(HttpContext context, Surface surface, OwnedRecord kind)
+ {
+ var request = await ProxyRequest.Repeat(context, Prefix(surface), context.RequestAborted);
+ var body = TryParse(request.Body);
+ var (ids, rebuild) = ExtractIds(body);
+
+ if (ids == null)
+ {
+ var broadcast = await gateway.SendAll(Members(surface), surface, request, context.RequestAborted);
+ await RelayOutcome(context, surface, broadcast);
+ return;
+ }
+
+ var resolution = await resolver.Resolve(context, kind, ids, context.RequestAborted);
+ if (resolution.Unresolved.Count > 0)
+ {
+ logger.LogWarning("No member holds {Count} of the {Kind} ids in {Method} {Path}; they were dropped: {Ids}",
+ resolution.Unresolved.Count, kind, context.Request.Method, context.Request.Path, string.Join(", ", resolution.Unresolved.Take(10)));
+ }
+
+ if (resolution.Owners.Count == 0)
+ {
+ await ProxyResponse.Problem(context, StatusCodes.Status404NotFound, $"No member holds any of the {kind} ids.", cancellationToken: context.RequestAborted);
+ return;
+ }
+
+ var responses = await Task.WhenAll(resolution.Owners.Select(owner =>
+ gateway.Send(owner.Key, surface, request with { Body = rebuild!(owner.Value), ContentType = "application/json" }, context.RequestAborted)));
+
+ await RelayOutcome(context, surface, responses);
+ }
+
+ static JsonNode? TryParse(byte[]? body)
+ {
+ if (body == null || body.Length == 0)
+ {
+ return null;
+ }
+
+ try
+ {
+ return JsonNode.Parse(body);
+ }
+ catch (JsonException)
+ {
+ return null;
+ }
+ }
+
+ static (List? Ids, Func, byte[]>? Rebuild) ExtractIds(JsonNode? body)
+ {
+ if (body is JsonArray array)
+ {
+ return ([.. array.Select(JsonValues.AsText).OfType()], ids => Serialize(new JsonArray([.. ids.Select(id => (JsonNode)id)])));
+ }
+
+ if (body is JsonObject obj)
+ {
+ var property = obj.FirstOrDefault(p => string.Equals(p.Key, "uniquemessageids", StringComparison.OrdinalIgnoreCase));
+ if (property.Value is JsonArray idArray)
+ {
+ byte[] Rebuild(List ids)
+ {
+ var copy = obj.DeepClone().AsObject();
+ copy[property.Key] = new JsonArray([.. ids.Select(id => (JsonNode)id)]);
+ return Serialize(copy);
+ }
+
+ return ([.. idArray.Select(JsonValues.AsText).OfType()], Rebuild);
+ }
+ }
+
+ return (null, null);
+ }
+
+ static byte[] Serialize(JsonNode node) => Encoding.UTF8.GetBytes(node.ToJsonString());
+
+ public const int DefaultPageSize = 50;
+}
diff --git a/src/ServiceControl.Federation/Api/InstanceNames.cs b/src/ServiceControl.Federation/Api/InstanceNames.cs
new file mode 100644
index 0000000000..c80d5166c8
--- /dev/null
+++ b/src/ServiceControl.Federation/Api/InstanceNames.cs
@@ -0,0 +1,46 @@
+namespace ServiceControl.Federation.Api;
+
+using System;
+using System.Text.Json.Nodes;
+using Aggregation;
+using Members;
+
+///
+/// Qualifies ServiceControl instance names with the member they belong to. Installations usually keep
+/// the default instance names, and ServicePulse attributes internal custom checks to platform health
+/// rows by name alone, so unqualified names would attach one installation's checks to every
+/// installation's rows.
+///
+static class InstanceNames
+{
+ public static string Qualify(string name, Member member)
+ {
+ var suffix = $" ({member.Name})";
+ return name.EndsWith(suffix, StringComparison.Ordinal) ? name : name + suffix;
+ }
+
+ ///
+ /// Qualifies host.instance_name in a configuration document.
+ ///
+ public static void QualifyConfiguration(JsonNode? configuration, Member member)
+ {
+ if (JsonValues.At(configuration, "host") is JsonObject host && JsonValues.Text(host, "instance_name") is { } name)
+ {
+ host["instance_name"] = Qualify(name, member);
+ }
+ }
+
+ ///
+ /// Qualifies the originating endpoint of a check the member's own instances report, so it keeps
+ /// matching the qualified instance name.
+ ///
+ public static void QualifyInternalCheck(JsonNode row, Member member)
+ {
+ if (JsonValues.Bool(row, "internal") == true
+ && JsonValues.At(row, "originating_endpoint") is JsonObject endpoint
+ && JsonValues.Text(endpoint, "name") is { } name)
+ {
+ endpoint["name"] = Qualify(name, member);
+ }
+ }
+}
diff --git a/src/ServiceControl.Federation/Api/LicensingHandlers.cs b/src/ServiceControl.Federation/Api/LicensingHandlers.cs
new file mode 100644
index 0000000000..9ea6ad4ebc
--- /dev/null
+++ b/src/ServiceControl.Federation/Api/LicensingHandlers.cs
@@ -0,0 +1,210 @@
+namespace ServiceControl.Federation.Api;
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net.Http;
+using System.Text.Json.Nodes;
+using System.Threading.Tasks;
+using Aggregation;
+using Http;
+using Members;
+using Microsoft.AspNetCore.Http;
+
+///
+/// Licensing and usage endpoints (api/licensing/*), which describe each installation separately.
+///
+static class LicensingHandlers
+{
+ const Surface Primary = Surface.Primary;
+
+ ///
+ /// Detected endpoints with their throughput, merged by queue identity. Daily maxima are summed,
+ /// which over-estimates when installations peak on different days; members do not expose the
+ /// daily series needed to do better.
+ ///
+ public static async Task Endpoints(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses))
+ {
+ return;
+ }
+
+ var merged = responses
+ .Where(r => r.IsSuccess)
+ .SelectMany(r => r.Array().OfType())
+ .GroupBy(row => $"{JsonValues.Text(row, "name_hash")}|{JsonValues.Text(row, "scope_hash")}", StringComparer.Ordinal)
+ .Select(group => MergeEndpoint([.. group]));
+
+ await ProxyResponse.WriteJson(context, new JsonArray([.. merged]), f.Meta(Primary, responses), context.RequestAborted);
+ }
+
+ static JsonObject MergeEndpoint(IReadOnlyList rows)
+ {
+ var result = rows[0].DeepClone().AsObject();
+ if (rows.Count == 1)
+ {
+ return result;
+ }
+
+ result["max_daily_throughput"] = rows.Sum(r => (long)(JsonValues.Number(r, "max_daily_throughput") ?? 0));
+ result["is_known_endpoint"] = rows.Any(r => JsonValues.Bool(r, "is_known_endpoint") == true);
+
+ if (rows.Any(r => JsonValues.At(r, "average_monthly_throughput") != null))
+ {
+ result["average_monthly_throughput"] = rows.Sum(r => (long)(JsonValues.Number(r, "average_monthly_throughput") ?? 0));
+ }
+
+ if (rows.Any(r => JsonValues.At(r, "monthly_throughput") is JsonArray))
+ {
+ result["monthly_throughput"] = new JsonArray([.. rows
+ .SelectMany(r => JsonValues.At(r, "monthly_throughput") as JsonArray ?? [])
+ .OfType()
+ .GroupBy(m => JsonValues.Text(m, "month") ?? "", StringComparer.Ordinal)
+ .OrderBy(g => g.Key, StringComparer.Ordinal)
+ .Select(g => new JsonObject
+ {
+ ["month"] = g.Key,
+ ["throughput"] = g.Sum(m => (long)(JsonValues.Number(m, "throughput") ?? 0))
+ })]);
+ }
+
+ return result;
+ }
+
+ ///
+ /// A report can be generated only when every installation can generate its own.
+ ///
+ public static async Task ReportAvailable(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses, notFoundIsAnAnswer: false))
+ {
+ return;
+ }
+
+ var successes = responses.Where(r => r.IsSuccess).ToList();
+ var result = successes[0].Json()!.DeepClone().AsObject();
+ var blocked = successes.FirstOrDefault(r => JsonValues.Bool(r.Json(), "report_can_be_generated") == false);
+ var unreachable = responses.FirstOrDefault(r => !r.IsSuccess);
+
+ if (blocked != null)
+ {
+ result["report_can_be_generated"] = false;
+ result["reason"] = $"{blocked.Member.Name}: {JsonValues.Text(blocked.Json(), "reason")}";
+ }
+ else if (unreachable != null)
+ {
+ result["report_can_be_generated"] = false;
+ result["reason"] = $"{unreachable.Member.Name} did not answer.";
+ }
+
+ await ProxyResponse.WriteJson(context, result, f.Meta(Primary, responses), context.RequestAborted);
+ }
+
+ ///
+ /// Connection tests pass only when they pass on every installation; error messages name the member.
+ ///
+ public static async Task SettingsTest(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses, notFoundIsAnAnswer: false))
+ {
+ return;
+ }
+
+ var successes = responses.Where(r => r.IsSuccess).ToList();
+ var result = successes[0].Json()!.DeepClone().AsObject();
+
+ foreach (var name in result.Select(p => p.Key).Where(k => k.EndsWith("_connection_result", StringComparison.Ordinal)).ToList())
+ {
+ var merged = result[name] as JsonObject ?? [];
+ merged["connection_successful"] = successes.All(r => JsonValues.Bool(r.Json(), $"{name}.connection_successful") != false)
+ && responses.All(r => r.IsSuccess);
+ merged["connection_error_messages"] = new JsonArray([.. successes
+ .SelectMany(r => (JsonValues.At(r.Json(), $"{name}.connection_error_messages") as JsonArray ?? [])
+ .Select(m => (JsonNode)$"{r.Member.Name}: {JsonValues.AsText(m)}"))
+ .Concat(responses.Where(r => !r.IsSuccess).Select(r => (JsonNode)$"{r.Member.Name} did not answer."))]);
+ result[name] = merged;
+ }
+
+ await ProxyResponse.WriteJson(context, result, f.Meta(Primary, responses), context.RequestAborted);
+ }
+
+ ///
+ /// Usage reports are signed per installation and cannot be merged. With one member the report is
+ /// relayed; with several, the signed reports are bundled into one zip, one entry per member.
+ ///
+ public static async Task ReportFile(Federator f, HttpContext context)
+ {
+ var members = f.Members(Primary);
+ var target = f.Target(context, Primary);
+ var request = new MemberRequest(HttpMethod.Get, target, ProxyRequest.Authorization(context), Accept: "*/*");
+
+ if (members.Count == 1)
+ {
+ using var single = await f.Gateway.SendStreaming(members[0], Primary, request, context.RequestAborted);
+ if (single == null)
+ {
+ await ProxyResponse.Problem(context, StatusCodes.Status502BadGateway, "The member did not answer.", cancellationToken: context.RequestAborted);
+ return;
+ }
+
+ await ProxyResponse.RelayStreaming(context, single, context.RequestAborted);
+ return;
+ }
+
+ var reports = new List<(Member Member, string FileName, byte[] Content)>();
+ var failures = new List<(Member Member, string Reason)>();
+ foreach (var member in members)
+ {
+ using var response = await f.Gateway.SendStreaming(member, Primary, request, context.RequestAborted);
+ if (response is not { IsSuccessStatusCode: true })
+ {
+ var reason = response == null ? "did not answer" : await response.Content.ReadAsStringAsync(context.RequestAborted);
+ failures.Add((member, reason));
+ continue;
+ }
+
+ var fileName = response.Content.Headers.ContentDisposition?.FileNameStar
+ ?? response.Content.Headers.ContentDisposition?.FileName?.Trim('"')
+ ?? "throughput-report.zip";
+ reports.Add((member, fileName, await response.Content.ReadAsByteArrayAsync(context.RequestAborted)));
+ }
+
+ if (reports.Count == 0)
+ {
+ context.Response.StatusCode = StatusCodes.Status400BadRequest;
+ context.Response.ContentType = "text/plain";
+ await context.Response.WriteAsync(string.Join(Environment.NewLine, failures.Select(failure => $"{failure.Member.Name}: {failure.Reason}")), context.RequestAborted);
+ return;
+ }
+
+ using var bundle = new MemoryStream();
+ using (var archive = new ZipArchive(bundle, ZipArchiveMode.Create, leaveOpen: true))
+ {
+ foreach (var (member, fileName, content) in reports)
+ {
+ var entry = archive.CreateEntry($"{member.Key}-{fileName}", CompressionLevel.NoCompression);
+ using var stream = entry.Open();
+ stream.Write(content);
+ }
+ }
+
+ var stamp = DateTime.UtcNow.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture);
+ context.Response.StatusCode = StatusCodes.Status200OK;
+ context.Response.ContentType = "application/zip";
+ context.Response.Headers.ContentDisposition = $"attachment; filename=\"federated.throughput-reports-{stamp}.zip\"";
+ if (failures.Count > 0)
+ {
+ context.Response.Headers[ProxyResponse.IncompleteResultsHeader] = string.Join(", ", failures.Select(failure => $"{failure.Member.PrimaryUri.ToString().TrimEnd('/')}:error"));
+ }
+
+ bundle.Position = 0;
+ await bundle.CopyToAsync(context.Response.Body, context.RequestAborted);
+ }
+}
diff --git a/src/ServiceControl.Federation/Api/MonitoringHandlers.cs b/src/ServiceControl.Federation/Api/MonitoringHandlers.cs
new file mode 100644
index 0000000000..92e116f14b
--- /dev/null
+++ b/src/ServiceControl.Federation/Api/MonitoringHandlers.cs
@@ -0,0 +1,142 @@
+namespace ServiceControl.Federation.Api;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Text.Json.Nodes;
+using System.Threading.Tasks;
+using Aggregation;
+using Http;
+using Members;
+using Microsoft.AspNetCore.Http;
+
+///
+/// The Monitoring surface. Members without a Monitoring URL do not take part.
+///
+static class MonitoringHandlers
+{
+ const Surface Monitoring = Surface.Monitoring;
+
+ static readonly HashSet HistoryPeriods = [1, 5, 10, 15, 30, 60];
+
+ public static async Task Root(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Monitoring, "");
+ if (await f.NothingAnswered(context, Monitoring, responses, notFoundIsAnAnswer: false))
+ {
+ return;
+ }
+
+ var source = responses.First(r => r.IsSuccess);
+ var body = source.Json()?.DeepClone() as JsonObject ?? [];
+ PrimaryHandlers.DescribeSource(body, source);
+
+ // ServicePulse has a single Monitoring row, so it carries the health of every member's Monitoring instance.
+ body["platform_health_status"] = PrimaryHandlers.WorstHealth(responses);
+
+ await ProxyResponse.WriteJson(context, body, new ResponseMeta { Version = f.LowestVersion(Monitoring) }, context.RequestAborted);
+ }
+
+ ///
+ /// ServicePulse reads Allow to decide whether instances can be removed, so only operations
+ /// every member supports are advertised.
+ ///
+ public static async Task Options(Federator f, HttpContext context)
+ {
+ var responses = await f.Gateway.SendAll(f.Members(Monitoring), Monitoring, new MemberRequest(HttpMethod.Options, "", ProxyRequest.Authorization(context)), context.RequestAborted);
+ var allowed = responses
+ .Where(r => r.IsSuccess)
+ .Select(r => (r.Header("Allow") ?? "").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet(StringComparer.OrdinalIgnoreCase))
+ .DefaultIfEmpty([])
+ .Aggregate((a, b) =>
+ {
+ a.IntersectWith(b);
+ return a;
+ });
+
+ context.Response.Headers.Allow = string.Join(", ", allowed);
+ context.Response.Headers.AccessControlExposeHeaders = "Allow";
+ var version = f.LowestVersion(Monitoring);
+ if (version != null)
+ {
+ context.Response.Headers["X-Particular-Version"] = version;
+ }
+
+ context.Response.StatusCode = StatusCodes.Status200OK;
+ }
+
+ public static async Task Endpoints(Federator f, HttpContext context)
+ {
+ if (!await ValidHistory(context))
+ {
+ return;
+ }
+
+ var responses = await f.FanOut(context, Monitoring);
+ if (await f.NothingAnswered(context, Monitoring, responses))
+ {
+ return;
+ }
+
+ var merged = MonitoredEndpointMerger.MergeList(Lists(responses));
+ await ProxyResponse.WriteJson(context, merged, f.Meta(Monitoring, responses), context.RequestAborted);
+ }
+
+ public static async Task EndpointDetails(Federator f, HttpContext context)
+ {
+ if (!await ValidHistory(context))
+ {
+ return;
+ }
+
+ var responses = await f.FanOut(context, Monitoring);
+ if (await f.NothingAnswered(context, Monitoring, responses))
+ {
+ return;
+ }
+
+ var details = responses.Where(r => r.IsSuccess).Select(r => r.Json()).OfType().ToList();
+ var merged = MonitoredEndpointMerger.MergeDetails(details);
+ await ProxyResponse.WriteJson(context, merged, f.Meta(Monitoring, responses), context.RequestAborted);
+ }
+
+ ///
+ /// Recomputed from the merged list: an endpoint is disconnected when every instance on every member
+ /// is stale. Summing the members' counts would be wrong once endpoints are merged by name.
+ ///
+ public static async Task Disconnected(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Monitoring, "monitored-endpoints?history=1");
+ if (await f.NothingAnswered(context, Monitoring, responses))
+ {
+ return;
+ }
+
+ var merged = MonitoredEndpointMerger.MergeList(Lists(responses));
+ var disconnected = merged.Count(endpoint => JsonValues.Bool(endpoint, "isStale") == true);
+ await ProxyResponse.WriteJson(context, JsonValue.Create(disconnected), f.Meta(Monitoring, responses), context.RequestAborted);
+ }
+
+ public static Task MyRoutes(Federator f, HttpContext context) => PrimaryHandlers.MyRoutes(f, context, Monitoring, "api/my/routes");
+
+ public static Task License(Federator f, HttpContext context) => PrimaryHandlers.License(f, context, Monitoring);
+
+ static List> Lists(IEnumerable responses) =>
+ [.. responses.Select(r => (IReadOnlyList)[.. r.IsSuccess ? r.Array().OfType() : []])];
+
+ ///
+ /// Monitoring throws on unknown history periods; answer 400 instead of fanning out a 500.
+ ///
+ static async Task ValidHistory(HttpContext context)
+ {
+ var history = ProxyRequest.Query(context, "history");
+ if (history == null || (int.TryParse(history, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var minutes) && HistoryPeriods.Contains(minutes)))
+ {
+ return true;
+ }
+
+ await ProxyResponse.Problem(context, StatusCodes.Status400BadRequest, $"Unknown history period '{history}'. Use one of {string.Join(", ", HistoryPeriods)}.", cancellationToken: context.RequestAborted);
+ return false;
+ }
+}
diff --git a/src/ServiceControl.Federation/Api/MonitoringRoutes.cs b/src/ServiceControl.Federation/Api/MonitoringRoutes.cs
new file mode 100644
index 0000000000..2cccbd3d6f
--- /dev/null
+++ b/src/ServiceControl.Federation/Api/MonitoringRoutes.cs
@@ -0,0 +1,30 @@
+namespace ServiceControl.Federation.Api;
+
+using System.Collections.Generic;
+using Members;
+using S = Strategy;
+
+///
+/// Every Monitoring instance route, relative to the Monitoring prefix, with the strategy that answers it.
+///
+public static class MonitoringRoutes
+{
+ public static readonly IReadOnlyList All =
+ [
+ Route(["GET"], "", S.Synthesised, MonitoringHandlers.Root) with { Anonymous = true },
+ Route(["OPTIONS"], "", S.Synthesised, MonitoringHandlers.Options) with { Anonymous = true },
+ Route(["GET"], "monitored-endpoints", S.KeyMerge, MonitoringHandlers.Endpoints),
+ Route(["GET"], "monitored-endpoints/disconnected", S.KeyMerge, MonitoringHandlers.Disconnected),
+ Route(["GET"], "monitored-endpoints/{endpointName}", S.KeyMerge, MonitoringHandlers.EndpointDetails),
+
+ // Removal only drops the instance from the member's in-memory registry, which is harmless on a
+ // member that does not know it, so it is broadcast rather than routed.
+ Route(["DELETE"], "monitored-instance/{endpointName}/{instanceId}", S.Broadcast, (f, c) => f.Broadcast(c, Surface.Monitoring)),
+
+ Route(["GET"], "connection", S.Lead, (f, c) => f.Lead(c, Surface.Monitoring)),
+ Route(["GET"], "license", S.Synthesised, MonitoringHandlers.License),
+ Route(["GET"], "api/my/routes", S.Synthesised, MonitoringHandlers.MyRoutes),
+ ];
+
+ static FederatedRoute Route(string[] methods, string template, string strategy, RouteHandler handler) => new(Surface.Monitoring, methods, template, strategy, handler);
+}
diff --git a/src/ServiceControl.Federation/Api/OwnerResolver.cs b/src/ServiceControl.Federation/Api/OwnerResolver.cs
new file mode 100644
index 0000000000..0087a3ec07
--- /dev/null
+++ b/src/ServiceControl.Federation/Api/OwnerResolver.cs
@@ -0,0 +1,110 @@
+namespace ServiceControl.Federation.Api;
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Configuration;
+using Http;
+using Members;
+using Microsoft.AspNetCore.Http;
+using Routing;
+
+///
+/// Finds the members that hold records addressed by id. Commands must only reach the owner, because
+/// members do not tolerate unknown ids (for example ArchiveMessageHandler dereferences the
+/// failed message without a null check).
+///
+public sealed class OwnerResolver(MemberRegistry registry, MemberGateway gateway, OwnerCache cache, FederationSettings settings)
+{
+ public async Task Resolve(HttpContext context, OwnedRecord kind, IReadOnlyCollection ids, CancellationToken cancellationToken = default)
+ {
+ var missing = ids.Where(id => cache.Find(kind, id) == null).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
+ if (missing.Count > 0)
+ {
+ await Probe(context, kind, missing, cancellationToken);
+ }
+
+ var owners = new Dictionary>();
+ var unresolved = new List();
+ foreach (var id in ids)
+ {
+ var memberKeys = cache.Find(kind, id);
+ if (memberKeys == null)
+ {
+ unresolved.Add(id);
+ continue;
+ }
+
+ foreach (var member in memberKeys.Select(registry.Find).OfType())
+ {
+ if (!owners.TryGetValue(member, out var memberIds))
+ {
+ memberIds = [];
+ owners[member] = memberIds;
+ }
+
+ memberIds.Add(id);
+ }
+ }
+
+ return new OwnerResolution(owners, unresolved);
+ }
+
+ Task Probe(HttpContext context, OwnedRecord kind, List ids, CancellationToken cancellationToken)
+ {
+ var members = registry.Serving(Surface.Primary);
+ var maxRows = settings.DeepPagingMaxRows.ToString(CultureInfo.InvariantCulture);
+
+ return kind switch
+ {
+ OwnedRecord.FailedMessage => ProbeEach(context, members, ids, id => $"errors/last/{Uri.EscapeDataString(id)}", cancellationToken),
+ OwnedRecord.CustomCheck => ProbeLists(context, members, kind, $"customchecks?per_page={maxRows}", "id", cancellationToken),
+ OwnedRecord.Endpoint => ProbeLists(context, members, kind, "endpoints", "id", cancellationToken),
+ OwnedRecord.Redirect => ProbeLists(context, members, kind, $"redirects?per_page={maxRows}", "message_redirect_id", cancellationToken),
+ _ => Task.CompletedTask
+ };
+ }
+
+ async Task ProbeEach(HttpContext context, IReadOnlyList members, List ids, Func target, CancellationToken cancellationToken)
+ {
+ using var throttle = new SemaphoreSlim(MaxConcurrentProbes);
+
+ async Task ProbeOne(Member member, string id)
+ {
+ await throttle.WaitAsync(cancellationToken);
+ try
+ {
+ var response = await gateway.Send(member, Surface.Primary, ProxyRequest.Get(context, target(id)), cancellationToken);
+ if (response.IsSuccess)
+ {
+ cache.Record(OwnedRecord.FailedMessage, id, member.Key);
+ }
+ }
+ finally
+ {
+ throttle.Release();
+ }
+ }
+
+ await Task.WhenAll(ids.SelectMany(id => members.Select(member => ProbeOne(member, id))));
+ }
+
+ async Task ProbeLists(HttpContext context, IReadOnlyList members, OwnedRecord kind, string target, string idField, CancellationToken cancellationToken)
+ {
+ var responses = await gateway.SendAll(members, Surface.Primary, ProxyRequest.Get(context, target), cancellationToken);
+ foreach (var response in responses.Where(r => r.IsSuccess))
+ {
+ foreach (var row in response.Array())
+ {
+ cache.Record(kind, Aggregation.JsonValues.Text(row, idField), response.Member.Key);
+ }
+ }
+ }
+
+ const int MaxConcurrentProbes = 16;
+}
+
+public sealed record OwnerResolution(IReadOnlyDictionary> Owners, IReadOnlyList Unresolved);
diff --git a/src/ServiceControl.Federation/Api/PrimaryHandlers.cs b/src/ServiceControl.Federation/Api/PrimaryHandlers.cs
new file mode 100644
index 0000000000..887cd72671
--- /dev/null
+++ b/src/ServiceControl.Federation/Api/PrimaryHandlers.cs
@@ -0,0 +1,661 @@
+namespace ServiceControl.Federation.Api;
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Net.Http;
+using System.Text.Json.Nodes;
+using System.Threading.Tasks;
+using Aggregation;
+using Http;
+using Members;
+using Microsoft.AspNetCore.Http;
+using Routing;
+
+///
+/// Route-specific merges for the primary API, on top of the strategies in .
+///
+static class PrimaryHandlers
+{
+ const Surface Primary = Surface.Primary;
+
+ ///
+ /// The root document: the lead member's, with every absolute URL pointing at the proxy.
+ ///
+ public static async Task Root(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary, "");
+ if (await f.NothingAnswered(context, Primary, responses, notFoundIsAnAnswer: false))
+ {
+ return;
+ }
+
+ var successes = responses.Where(r => r.IsSuccess).ToList();
+ var source = successes[0];
+ var body = source.Json()!.DeepClone().AsObject();
+ var memberBase = source.Member.PrimaryUri.ToString();
+ var proxyBase = ProxyRequest.BaseUrl(context, f.Prefix(Primary));
+
+ foreach (var property in body.ToList())
+ {
+ if (JsonValues.AsText(property.Value) is { } text && text.StartsWith(memberBase, StringComparison.OrdinalIgnoreCase))
+ {
+ body[property.Key] = proxyBase + text[memberBase.Length..];
+ }
+ }
+
+ body["name"] = f.Settings.DisplayName;
+
+ var invalidLicense = successes.Select(r => JsonValues.Text(r.Json(), "license_status")).FirstOrDefault(s => s is not null and not "valid");
+ if (invalidLicense != null)
+ {
+ body["license_status"] = invalidLicense;
+ }
+
+ // The other member primaries have their own rows (configuration/remotes), so this row describes only the member that answered.
+ DescribeSource(body, source);
+
+ await ProxyResponse.WriteJson(context, body, new ResponseMeta { Version = f.LowestVersion(Primary) }, context.RequestAborted);
+ }
+
+ ///
+ /// ServicePulse shows a root document's platform_health_version in preference to
+ /// X-Particular-Version, while its feature gates read only the header. The body therefore
+ /// carries the answering member's own version, and the header keeps the lowest member version.
+ ///
+ public static void DescribeSource(JsonObject body, MemberResponse source) =>
+ body["platform_health_version"] = JsonValues.Text(source.Json(), "platform_health_version") ?? source.Version ?? JsonValues.Text(source.Json(), "version");
+
+ ///
+ /// The worst health any member reports, and at least degraded when a member does not answer.
+ ///
+ public static string WorstHealth(IReadOnlyList responses)
+ {
+ var worst = responses
+ .Where(r => r.IsSuccess)
+ .Select(r => JsonValues.Text(r.Json(), "platform_health_status") ?? "healthy")
+ .DefaultIfEmpty("healthy")
+ .MaxBy(HealthRank)!;
+
+ return (responses.Any(r => !r.IsSuccess) && HealthRank(worst) < HealthRank("degraded")) ? "degraded" : worst;
+ }
+
+ static int HealthRank(string status) => status switch
+ {
+ "healthy" => 0,
+ "degraded" => 1,
+ "unavailable" => 2,
+ _ => 1
+ };
+
+ ///
+ /// configuration and instance-info: the lead member's document, with the shortest
+ /// retention periods, since ServicePulse uses them to warn about data scheduled for deletion.
+ ///
+ public static async Task Configuration(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses, notFoundIsAnAnswer: false))
+ {
+ return;
+ }
+
+ var source = responses.First(r => r.IsSuccess);
+ var successes = responses.Where(r => r.IsSuccess).Select(r => r.Json()).ToList();
+ var body = source.Json()!.DeepClone().AsObject();
+
+ if (body["data_retention"] is JsonObject retention)
+ {
+ foreach (var field in new[] { "error_retention_period", "audit_retention_period" })
+ {
+ var shortest = successes
+ .Select(s => JsonValues.Text(s, $"data_retention.{field}"))
+ .Where(v => v != null && TimeSpan.TryParse(v, CultureInfo.InvariantCulture, out _))
+ .MinBy(v => TimeSpan.Parse(v!, CultureInfo.InvariantCulture));
+ if (shortest != null)
+ {
+ retention[field] = shortest;
+ }
+ }
+ }
+
+ InstanceNames.QualifyConfiguration(body, source.Member);
+
+ await ProxyResponse.WriteJson(context, body, new ResponseMeta { Version = f.LowestVersion(Primary) }, context.RequestAborted);
+ }
+
+ ///
+ /// The members' audit instances, plus every non-lead member primary as a remote error instance,
+ /// which ServicePulse already shows in platform health.
+ ///
+ public static async Task Remotes(Federator f, HttpContext context)
+ {
+ var members = f.Members(Primary);
+ var remotesTask = f.FanOut(context, Primary, "configuration/remotes");
+ var configurationsTask = f.Gateway.SendAll(members.Skip(1), Primary, ProxyRequest.Get(context, "configuration"), context.RequestAborted);
+ var remotes = await remotesTask;
+ var configurations = await configurationsTask;
+
+ // Not de-duplicated by api_uri: it is the address the member's own primary uses, so two
+ // installations can list different audit instances under the same one.
+ var result = new JsonArray();
+ foreach (var response in remotes.Where(r => r.IsSuccess))
+ {
+ foreach (var remote in response.Array().OfType())
+ {
+ var entry = remote.DeepClone();
+ InstanceNames.QualifyConfiguration(JsonValues.At(entry, "configuration"), response.Member);
+ result.Add(entry);
+ }
+ }
+
+ foreach (var configuration in configurations)
+ {
+ var member = configuration.Member;
+ var entry = new JsonObject
+ {
+ ["api_uri"] = member.PrimaryUri.ToString(),
+ ["version"] = member.Version(Primary) ?? "Unknown",
+ ["status"] = configuration.IsSuccess ? "online" : "unavailable"
+ };
+ if (configuration.IsSuccess)
+ {
+ var memberConfiguration = configuration.Json()?.DeepClone();
+ InstanceNames.QualifyConfiguration(memberConfiguration, member);
+ entry["configuration"] = memberConfiguration;
+ }
+
+ result.Add(entry);
+ }
+
+ await ProxyResponse.WriteJson(context, result, new ResponseMeta { Version = f.LowestVersion(Primary) }, context.RequestAborted);
+ }
+
+ ///
+ /// The routes the caller may invoke: those allowed on every member that answered, because
+ /// broadcast commands reach every member.
+ ///
+ public static Task MyRoutes(Federator f, HttpContext context) => MyRoutes(f, context, Primary, "my/routes");
+
+ public static async Task MyRoutes(Federator f, HttpContext context, Surface surface, string memberTarget)
+ {
+ var responses = await f.FanOut(context, surface, memberTarget);
+ if (await f.NothingAnswered(context, surface, responses, notFoundIsAnAnswer: false))
+ {
+ return;
+ }
+
+ var successes = responses.Where(r => r.IsSuccess).Select(r => r.Json()).ToList();
+ var allowedEverywhere = successes
+ .Select(s => (JsonValues.At(s, "routes") as JsonArray ?? []).Select(RouteKey).ToHashSet(StringComparer.Ordinal))
+ .Aggregate((a, b) =>
+ {
+ a.IntersectWith(b);
+ return a;
+ });
+
+ var routes = new JsonArray([.. (JsonValues.At(successes[0], "routes") as JsonArray ?? [])
+ .Where(route => allowedEverywhere.Contains(RouteKey(route)))
+ .Select(route => route?.DeepClone())]);
+
+ var roles = new JsonArray([.. successes
+ .SelectMany(s => JsonValues.At(s, "roles") as JsonArray ?? [])
+ .Select(JsonValues.AsText)
+ .OfType()
+ .Distinct(StringComparer.Ordinal)
+ .Select(role => (JsonNode)role)]);
+
+ await ProxyResponse.WriteJson(context, new JsonObject { ["roles"] = roles, ["routes"] = routes }, new ResponseMeta { Version = f.LowestVersion(surface) }, context.RequestAborted);
+ }
+
+ static string RouteKey(JsonNode? route)
+ {
+ var method = (JsonValues.Text(route, "method") ?? "").ToUpperInvariant();
+ var template = JsonValues.Text(route, "url_template") ?? "";
+ var normalised = string.Join('/', template.Trim('/').Split('/').Select(s => s.StartsWith('{') ? "{}" : s.ToLowerInvariant()));
+ return $"{method} /{normalised}";
+ }
+
+ ///
+ /// The license of the member in the worst state, so an expiring or invalid license anywhere is visible.
+ ///
+ public static Task License(Federator f, HttpContext context) => License(f, context, Primary);
+
+ public static async Task License(Federator f, HttpContext context, Surface surface)
+ {
+ var responses = await f.FanOut(context, surface);
+ if (await f.NothingAnswered(context, surface, responses, notFoundIsAnAnswer: false))
+ {
+ return;
+ }
+
+ var successes = responses.Where(r => r.IsSuccess).Select(r => r.Json()).ToList();
+ var worst = successes.FirstOrDefault(s => string.Equals(JsonValues.Text(s, "status"), "invalid", StringComparison.OrdinalIgnoreCase))
+ ?? successes.FirstOrDefault(s => JsonValues.Text(s, "license_status") is { } status && !string.Equals(status, "Valid", StringComparison.OrdinalIgnoreCase))
+ ?? successes[0];
+
+ var body = worst!.DeepClone().AsObject();
+ if (body.ContainsKey("instance_name"))
+ {
+ body["instance_name"] = f.Settings.DisplayName;
+ }
+
+ await ProxyResponse.WriteJson(context, body, new ResponseMeta { Version = f.LowestVersion(surface) }, context.RequestAborted);
+ }
+
+ ///
+ /// Failed message lists: exact sorted page merge, recording owners for later commands.
+ ///
+ public static Task FailedMessages(Federator f, HttpContext context) =>
+ f.SortedPage(
+ context,
+ Primary,
+ RowOrder.FailedMessages(ProxyRequest.Query(context, "sort"), ProxyRequest.Query(context, "direction")),
+ OwnedRecord.FailedMessage);
+
+ ///
+ /// Audit and composite message lists. Rows are de-duplicated as ServiceControl does, and every
+ /// instance_id is made routable back to its member.
+ ///
+ public static Task Messages(Federator f, HttpContext context) =>
+ f.SortedPage(
+ context,
+ Primary,
+ RowOrder.Messages(ProxyRequest.Query(context, "sort"), ProxyRequest.Query(context, "direction")),
+ deduplicationKey: MessageKey,
+ transform: (row, member) => InstanceIdCodec.RewriteTree(row, member.Key));
+
+ public static Task Messages2(Federator f, HttpContext context) =>
+ f.TopN(
+ context,
+ Primary,
+ RowOrder.Messages(ProxyRequest.Query(context, "sort"), ProxyRequest.Query(context, "direction")),
+ "page_size",
+ MessageKey,
+ (row, member) => InstanceIdCodec.RewriteTree(row, member.Key));
+
+ static string? MessageKey(JsonNode row) => $"{JsonValues.Text(row, "receiving_endpoint.name")}-{JsonValues.Text(row, "message_id")}";
+
+ public static Task CustomChecks(Federator f, HttpContext context) =>
+ f.SortedPage(context, Primary, RowOrder.CustomChecks(), OwnedRecord.CustomCheck, transform: InstanceNames.QualifyInternalCheck);
+
+ ///
+ /// EF Core persisters number event log items 1, 2, 3… on every member, so ids are prefixed with
+ /// the member key. No API accepts an event log id, so nothing needs to decode it.
+ ///
+ public static Task EventLog(Federator f, HttpContext context) =>
+ f.SortedPage(
+ context,
+ Primary,
+ RowOrder.EventLog(),
+ transform: (row, member) =>
+ {
+ if (row is JsonObject obj && JsonValues.Text(obj, "id") is { } id)
+ {
+ obj["id"] = InstanceIdCodec.Encode(member.Key, id);
+ }
+ });
+
+ public static Task Redirects(Federator f, HttpContext context) =>
+ f.SortedPage(
+ context,
+ Primary,
+ RowOrder.Redirects(ProxyRequest.Query(context, "sort"), ProxyRequest.Query(context, "direction")),
+ OwnedRecord.Redirect,
+ "message_redirect_id");
+
+ public static Task Endpoints(Federator f, HttpContext context) => f.Union(context, Primary, kind: OwnedRecord.Endpoint);
+
+ public static Task KnownEndpoints(Federator f, HttpContext context) =>
+ f.Union(context, Primary, row => JsonValues.Text(row, "id"), OwnedRecord.Endpoint, totalFromRows: true);
+
+ public static Task EndpointSettings(Federator f, HttpContext context) => f.Union(context, Primary, row => JsonValues.Text(row, "name") ?? "");
+
+ public static Task Classifiers(Federator f, HttpContext context) => f.Union(context, Primary, JsonValues.AsText);
+
+ ///
+ /// Sums every numeric property; used for heartbeats/stats.
+ ///
+ public static async Task SumNumbers(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses))
+ {
+ return;
+ }
+
+ var result = new JsonObject();
+ foreach (var body in responses.Where(r => r.IsSuccess).Select(r => r.Json()).OfType())
+ {
+ foreach (var (name, value) in body)
+ {
+ var number = JsonValues.AsNumber(value);
+ if (number != null && JsonValues.AsNumber(result[name]) is { } existing)
+ {
+ result[name] = existing + number.Value;
+ }
+ else if (!result.ContainsKey(name))
+ {
+ result[name] = value?.DeepClone();
+ }
+ }
+ }
+
+ await ProxyResponse.WriteJson(context, result, f.Meta(Primary, responses, responses.Where(r => r.IsSuccess).Sum(r => (long)(r.TotalCount ?? 0))), context.RequestAborted);
+ }
+
+ ///
+ /// errors/summary: { category: { name: count } }, summed per name.
+ ///
+ public static async Task ErrorsSummary(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses))
+ {
+ return;
+ }
+
+ var result = new JsonObject();
+ foreach (var body in responses.Where(r => r.IsSuccess).Select(r => r.Json()).OfType())
+ {
+ foreach (var (category, counts) in body)
+ {
+ if (counts is not JsonObject countsByName)
+ {
+ continue;
+ }
+
+ if (result[category] is not JsonObject target)
+ {
+ target = [];
+ result[category] = target;
+ }
+
+ foreach (var (name, count) in countsByName)
+ {
+ target[name] = (long)((JsonValues.AsNumber(target[name]) ?? 0) + (JsonValues.AsNumber(count) ?? 0));
+ }
+ }
+ }
+
+ await ProxyResponse.WriteJson(context, result, f.Meta(Primary, responses), context.RequestAborted);
+ }
+
+ ///
+ /// Failed message counts per queue, summed per address and paged by the proxy.
+ ///
+ public static async Task QueueAddresses(Federator f, HttpContext context)
+ {
+ var maxRows = f.Settings.DeepPagingMaxRows.ToString(CultureInfo.InvariantCulture);
+ var target = ProxyRequest.WithQuery(context, f.Prefix(Primary), ProxyRequest.QueryWith(context, ("page", "1"), ("per_page", maxRows)));
+ var responses = await f.FanOut(context, Primary, target);
+ if (await f.NothingAnswered(context, Primary, responses))
+ {
+ return;
+ }
+
+ var merged = responses
+ .Where(r => r.IsSuccess)
+ .SelectMany(r => r.Array().OfType())
+ .GroupBy(row => JsonValues.Text(row, "physical_address") ?? "", StringComparer.OrdinalIgnoreCase)
+ .Select(group =>
+ {
+ var row = group.First().DeepClone().AsObject();
+ row["failed_message_count"] = group.Sum(r => (long)(JsonValues.Number(r, "failed_message_count") ?? 0));
+ return row;
+ })
+ .OrderBy(row => JsonValues.Text(row, "physical_address"), StringComparer.Ordinal)
+ .ToList();
+
+ var page = ProxyRequest.IntQuery(context, "page", 1);
+ var pageSize = ProxyRequest.IntQuery(context, "per_page", Federator.DefaultPageSize);
+ var rows = new JsonArray([.. merged.Skip((page - 1) * pageSize).Take(pageSize)]);
+ var link = ProxyResponse.PagingLinks(context, f.Prefix(Primary), page, pageSize, merged.Count);
+
+ await ProxyResponse.WriteJson(context, rows, f.Meta(Primary, responses, merged.Count, target, link), context.RequestAborted);
+ }
+
+ public static async Task AuditCounts(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses))
+ {
+ return;
+ }
+
+ var merged = responses
+ .Where(r => r.IsSuccess)
+ .SelectMany(r => r.Array().OfType())
+ .GroupBy(row => JsonValues.Text(row, "utc_date") ?? "", StringComparer.Ordinal)
+ .Select(group =>
+ {
+ var row = group.First().DeepClone().AsObject();
+ row["count"] = group.Sum(r => (long)(JsonValues.Number(r, "count") ?? 0));
+ return row;
+ })
+ .OrderBy(row => JsonValues.TimeTicks(row, "utc_date") ?? 0);
+
+ await ProxyResponse.WriteJson(context, new JsonArray([.. merged]), f.Meta(Primary, responses), context.RequestAborted);
+ }
+
+ ///
+ /// Unresolved and archived failure group lists, merged by group id.
+ ///
+ public static async Task Groups(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses))
+ {
+ return;
+ }
+
+ var merged = FailureGroupMerger.MergeLists([.. responses.Select(r => (IReadOnlyList)[.. r.Array().OfType()])]);
+ await ProxyResponse.WriteJson(context, merged, f.Meta(Primary, responses, merged.Count), context.RequestAborted);
+ }
+
+ public static async Task GroupById(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses))
+ {
+ return;
+ }
+
+ var hits = responses.Where(r => r.IsSuccess).Select(r => r.Json()).OfType().ToList();
+ if (hits.Count == 0)
+ {
+ await ProxyResponse.Problem(context, StatusCodes.Status404NotFound, "No member holds this group.", cancellationToken: context.RequestAborted);
+ return;
+ }
+
+ await ProxyResponse.WriteJson(context, FailureGroupMerger.Merge(hits), f.Meta(Primary, responses), context.RequestAborted);
+ }
+
+ ///
+ /// Retry history: operations from every member, newest first, keeping one member's depth.
+ ///
+ public static async Task History(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses))
+ {
+ return;
+ }
+
+ var bodies = responses.Where(r => r.IsSuccess).Select(r => r.Json()).OfType().ToList();
+ if (bodies.Count == 0)
+ {
+ await ProxyResponse.WriteJson(context, new JsonObject(), f.Meta(Primary, responses), context.RequestAborted);
+ return;
+ }
+
+ var result = bodies[0].DeepClone().AsObject();
+ var historic = bodies.Select(b => b["historic_operations"] as JsonArray ?? []).ToList();
+ var depth = historic.Max(h => h.Count);
+ result["historic_operations"] = new JsonArray([.. historic
+ .SelectMany(h => h.OfType())
+ .OrderByDescending(op => JsonValues.TimeTicks(op, "completion_time") ?? long.MinValue)
+ .Take(depth)
+ .Select(op => op.DeepClone())]);
+ result["unacknowledged_operations"] = new JsonArray([.. bodies
+ .SelectMany(b => b["unacknowledged_operations"] as JsonArray ?? [])
+ .OfType()
+ .Select(op => op.DeepClone())]);
+
+ await ProxyResponse.WriteJson(context, result, f.Meta(Primary, responses), context.RequestAborted);
+ }
+
+ ///
+ /// Saga history may be spread over members; changes are merged newest first, as
+ /// GetSagaByIdApi does across audit instances.
+ ///
+ public static async Task Saga(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses))
+ {
+ return;
+ }
+
+ var hits = responses.Where(r => r.IsSuccess && r.Json() != null).ToList();
+ if (hits.Count == 0)
+ {
+ await ProxyResponse.Problem(context, StatusCodes.Status404NotFound, "No member holds this saga.", cancellationToken: context.RequestAborted);
+ return;
+ }
+
+ foreach (var hit in hits)
+ {
+ InstanceIdCodec.RewriteTree(hit.Json(), hit.Member.Key);
+ }
+
+ var result = hits[0].Json()!.DeepClone().AsObject();
+ if (hits.Count > 1)
+ {
+ result["changes"] = new JsonArray([.. hits
+ .SelectMany(h => JsonValues.At(h.Json(), "changes") as JsonArray ?? [])
+ .OfType()
+ .OrderByDescending(c => JsonValues.TimeTicks(c, "finish_time") ?? long.MinValue)
+ .Select(c => c.DeepClone())]);
+ }
+
+ await ProxyResponse.WriteJson(context, result, f.Meta(Primary, responses), context.RequestAborted);
+ }
+
+ ///
+ /// Message bodies are routed by the member key in instance_id; without one, the first member
+ /// that has the body answers.
+ ///
+ public static async Task Body(Federator f, HttpContext context)
+ {
+ var accept = context.Request.Headers.Accept is { Count: > 0 } value ? value.ToString() : null;
+ var authorization = ProxyRequest.Authorization(context);
+
+ if (InstanceIdCodec.TryDecode(ProxyRequest.Query(context, "instance_id"), out var memberKey, out var memberInstanceId)
+ && f.Registry.Find(memberKey) is { } owner)
+ {
+ var target = TargetWithInstanceId(f, context, memberInstanceId);
+ using var response = await f.Gateway.SendStreaming(owner, Primary, new MemberRequest(HttpMethod.Get, target, authorization, Accept: accept), context.RequestAborted);
+ if (response == null)
+ {
+ await ProxyResponse.Problem(context, StatusCodes.Status502BadGateway, $"Member '{owner.Key}' did not answer.", cancellationToken: context.RequestAborted);
+ return;
+ }
+
+ await ProxyResponse.RelayStreaming(context, response, context.RequestAborted);
+ return;
+ }
+
+ var fallbackTarget = TargetWithInstanceId(f, context, null);
+ foreach (var member in f.Members(Primary))
+ {
+ using var response = await f.Gateway.SendStreaming(member, Primary, new MemberRequest(HttpMethod.Get, fallbackTarget, authorization, Accept: accept), context.RequestAborted);
+ if (response is { StatusCode: not System.Net.HttpStatusCode.NotFound })
+ {
+ await ProxyResponse.RelayStreaming(context, response, context.RequestAborted);
+ return;
+ }
+ }
+
+ await ProxyResponse.Problem(context, StatusCodes.Status404NotFound, "No member holds this message body.", cancellationToken: context.RequestAborted);
+ }
+
+ ///
+ /// A single retry may name the instance that holds the message; route by it when it does.
+ ///
+ public static async Task RetryById(Federator f, HttpContext context)
+ {
+ if (InstanceIdCodec.TryDecode(ProxyRequest.Query(context, "instance_id"), out var memberKey, out var memberInstanceId)
+ && f.Registry.Find(memberKey) is { } owner)
+ {
+ var request = await ProxyRequest.Repeat(context, f.Prefix(Primary), context.RequestAborted);
+ var response = await f.Gateway.Send(owner, Primary, request with { PathAndQuery = TargetWithInstanceId(f, context, memberInstanceId) }, context.RequestAborted);
+ await f.RelayOutcome(context, Primary, [response]);
+ return;
+ }
+
+ await f.RouteToOwners(context, Primary, OwnedRecord.FailedMessage, RouteValue(context, "failedMessageId"));
+ }
+
+ static string TargetWithInstanceId(Federator f, HttpContext context, string? memberInstanceId)
+ {
+ var query = ProxyRequest.QueryWith(context).Where(q => q.Key != "instance_id").ToList();
+ if (memberInstanceId != null)
+ {
+ query.Add(new KeyValuePair("instance_id", memberInstanceId));
+ }
+
+ return ProxyRequest.WithQuery(context, f.Prefix(Primary), query);
+ }
+
+ ///
+ /// A redirect belongs to the broker of the member whose failed messages came from the source
+ /// queue. When no member has failures from that queue, every member gets the redirect.
+ ///
+ public static async Task CreateRedirect(Federator f, HttpContext context)
+ {
+ var request = await ProxyRequest.Repeat(context, f.Prefix(Primary), context.RequestAborted);
+ var from = request.Body is { Length: > 0 } body ? JsonValues.Text(JsonNode.Parse(body), "fromphysicaladdress") : null;
+
+ var targets = f.Members(Primary);
+ if (from != null)
+ {
+ var maxRows = f.Settings.DeepPagingMaxRows.ToString(CultureInfo.InvariantCulture);
+ var queues = await f.FanOut(context, Primary, $"errors/queues/addresses?per_page={maxRows}");
+ var owners = queues
+ .Where(r => r.IsSuccess && r.Array().Any(q => string.Equals(JsonValues.Text(q, "physical_address"), from, StringComparison.OrdinalIgnoreCase)))
+ .Select(r => r.Member)
+ .ToList();
+ if (owners.Count > 0)
+ {
+ targets = owners;
+ }
+ }
+
+ var responses = await f.Gateway.SendAll(targets, Primary, request, context.RequestAborted);
+ await f.RelayOutcome(context, Primary, responses);
+ }
+
+ public static async Task RetentionStatus(Federator f, HttpContext context)
+ {
+ var responses = await f.FanOut(context, Primary);
+ if (await f.NothingAnswered(context, Primary, responses, notFoundIsAnAnswer: false))
+ {
+ return;
+ }
+
+ var successes = responses.Where(r => r.IsSuccess).Select(r => r.Json()).OfType().ToList();
+ if (successes.Count == 0)
+ {
+ await f.RelayOutcome(context, Primary, responses);
+ return;
+ }
+
+ var result = successes[0].DeepClone().AsObject();
+ result["is_running"] = successes.Any(s => JsonValues.Bool(s, "is_running") == true);
+ await ProxyResponse.WriteJson(context, result, f.Meta(Primary, responses), context.RequestAborted);
+ }
+
+ public static string RouteValue(HttpContext context, string name) => context.Request.RouteValues[name]?.ToString() ?? "";
+}
diff --git a/src/ServiceControl.Federation/Api/PrimaryRoutes.cs b/src/ServiceControl.Federation/Api/PrimaryRoutes.cs
new file mode 100644
index 0000000000..68d2cc2490
--- /dev/null
+++ b/src/ServiceControl.Federation/Api/PrimaryRoutes.cs
@@ -0,0 +1,148 @@
+namespace ServiceControl.Federation.Api;
+
+using System.Collections.Generic;
+using Members;
+using Routing;
+using S = Strategy;
+
+///
+/// Every primary (error) instance route, relative to /api, with the strategy that answers it.
+/// The route coverage test compares this table with the primary's approved route list, so a new
+/// ServiceControl route fails the build until it is classified here.
+///
+public static class PrimaryRoutes
+{
+ public static readonly IReadOnlyList All =
+ [
+ // Discovery, configuration and authorization
+ Get("", S.Synthesised, PrimaryHandlers.Root) with { Anonymous = true },
+ Get("configuration", S.Synthesised, PrimaryHandlers.Configuration) with { Anonymous = true },
+ Get("instance-info", S.Synthesised, PrimaryHandlers.Configuration) with { Anonymous = true },
+ Get("configuration/remotes", S.Union, PrimaryHandlers.Remotes) with { Anonymous = true },
+ Get("authentication/configuration", S.Lead, (f, c) => f.Lead(c, Surface.Primary)) with { Anonymous = true },
+ Get("connection", S.Lead, (f, c) => f.Lead(c, Surface.Primary)),
+ Get("my/routes", S.Synthesised, PrimaryHandlers.MyRoutes),
+
+ // License
+ Get("license", S.Synthesised, PrimaryHandlers.License),
+ Get("license/details", S.Lead, (f, c) => f.Lead(c, Surface.Primary)),
+ Post("license/detailsUpload", S.Lead, (f, c) => f.LeadStreaming(c, Surface.Primary)),
+ // Amends the details uploaded above, so it goes to the same member.
+ Post("license/amendDetails", S.Lead, (f, c) => f.Lead(c, Surface.Primary)),
+
+ // Failed messages
+ Get("errors", S.SortedPage, PrimaryHandlers.FailedMessages),
+ Head("errors", S.Sum, (f, c) => f.SumCount(c, Surface.Primary)),
+ Get("errors/summary", S.Sum, PrimaryHandlers.ErrorsSummary),
+ Get("errors/{failedMessageId}", S.FirstHit, (f, c) => f.FirstHit(c, Surface.Primary, OwnedRecord.FailedMessage)),
+ Get("errors/last/{failedMessageId}", S.FirstHit, (f, c) => f.FirstHit(c, Surface.Primary, OwnedRecord.FailedMessage)),
+ Get("errors/queues/addresses", S.KeyMerge, PrimaryHandlers.QueueAddresses),
+ Get("errors/groups/{classifier?}", S.KeyMerge, PrimaryHandlers.Groups),
+ Get("endpoints/{endpointname}/errors", S.SortedPage, PrimaryHandlers.FailedMessages),
+ Get("archive/groups/id/{groupId}", S.KeyMerge, PrimaryHandlers.GroupById),
+ Post("errors/retry", S.RouteToOwner, (f, c) => f.Partitioned(c, Surface.Primary, OwnedRecord.FailedMessage)),
+ Post("errors/retry/all", S.Broadcast, Broadcast),
+ Post("errors/{endpointName}/retry/all", S.Broadcast, Broadcast),
+ Post("errors/queues/{queueAddress}/retry", S.Broadcast, Broadcast),
+ Post("errors/{failedMessageId}/retry", S.RouteToOwner, PrimaryHandlers.RetryById),
+ Route(["PATCH", "POST"], "errors/archive", S.RouteToOwner, (f, c) => f.Partitioned(c, Surface.Primary, OwnedRecord.FailedMessage)),
+ Route(["PATCH", "POST"], "errors/{messageId}/archive", S.RouteToOwner, (f, c) => f.RouteToOwners(c, Surface.Primary, OwnedRecord.FailedMessage, PrimaryHandlers.RouteValue(c, "messageId"))),
+ Patch("errors/unarchive", S.RouteToOwner, (f, c) => f.Partitioned(c, Surface.Primary, OwnedRecord.FailedMessage)),
+ Patch("errors/{from}...{to}/unarchive", S.Broadcast, Broadcast),
+ Get("edit/config", S.Lead, (f, c) => f.Lead(c, Surface.Primary)),
+ Post("edit/{failedMessageId}", S.RouteToOwner, (f, c) => f.RouteToOwners(c, Surface.Primary, OwnedRecord.FailedMessage, PrimaryHandlers.RouteValue(c, "failedMessageId"))),
+
+ // Pending retries
+ Post("pendingretries/retry", S.RouteToOwner, (f, c) => f.Partitioned(c, Surface.Primary, OwnedRecord.FailedMessage)),
+ Patch("pendingretries/resolve", S.RouteToOwner, (f, c) => f.Partitioned(c, Surface.Primary, OwnedRecord.FailedMessage)),
+ Post("pendingretries/queues/retry", S.Broadcast, Broadcast),
+ Patch("pendingretries/queues/resolve", S.Broadcast, Broadcast),
+
+ // Failure groups
+ Get("recoverability/classifiers", S.Union, PrimaryHandlers.Classifiers),
+ Get("recoverability/groups/{classifier?}", S.KeyMerge, PrimaryHandlers.Groups),
+ Get("recoverability/groups/id/{groupId}", S.KeyMerge, PrimaryHandlers.GroupById),
+ Get("recoverability/groups/{groupId}/errors", S.SortedPage, PrimaryHandlers.FailedMessages),
+ Head("recoverability/groups/{groupId}/errors", S.Sum, (f, c) => f.SumCount(c, Surface.Primary)),
+ Post("recoverability/groups/{groupId}/errors/retry", S.Broadcast, Broadcast),
+ Post("recoverability/groups/{groupId}/errors/archive", S.Broadcast, Broadcast),
+ Post("recoverability/groups/{groupId}/errors/unarchive", S.Broadcast, Broadcast),
+ Post("recoverability/groups/{groupId}/comment", S.Broadcast, Broadcast),
+ Delete("recoverability/groups/{groupId}/comment", S.Broadcast, Broadcast),
+ Get("recoverability/history", S.Union, PrimaryHandlers.History),
+ Delete("recoverability/unacknowledgedgroups/{groupId}", S.Broadcast, Broadcast),
+
+ // Audit and composite message views
+ Get("messages", S.SortedPage, PrimaryHandlers.Messages),
+ Get("messages/search", S.SortedPage, PrimaryHandlers.Messages),
+ Get("messages/search/{keyword}", S.SortedPage, PrimaryHandlers.Messages),
+ Get("messages/{id}/body", S.RouteToOwner, PrimaryHandlers.Body),
+ Get("messages2", S.TopN, PrimaryHandlers.Messages2),
+ Get("endpoints/{endpoint}/messages", S.SortedPage, PrimaryHandlers.Messages),
+ Get("endpoints/{endpoint}/messages/search", S.SortedPage, PrimaryHandlers.Messages),
+ Get("endpoints/{endpoint}/messages/search/{keyword}", S.SortedPage, PrimaryHandlers.Messages),
+ Get("endpoints/{endpoint}/audit-count", S.KeyMerge, PrimaryHandlers.AuditCounts),
+ Get("conversations/{conversationId}", S.SortedPage, PrimaryHandlers.Messages),
+ Get("sagas/{id}", S.KeyMerge, PrimaryHandlers.Saga),
+
+ // Endpoints and heartbeats
+ Get("endpoints", S.Union, PrimaryHandlers.Endpoints),
+ Options("endpoints", S.Lead, (f, c) => f.Lead(c, Surface.Primary)),
+ Get("endpoints/known", S.Union, PrimaryHandlers.KnownEndpoints),
+ Patch("endpoints/{endpointId}", S.RouteToOwner, (f, c) => f.RouteToOwners(c, Surface.Primary, OwnedRecord.Endpoint, PrimaryHandlers.RouteValue(c, "endpointId"))),
+ Delete("endpoints/{endpointId}", S.RouteToOwner, (f, c) => f.RouteToOwners(c, Surface.Primary, OwnedRecord.Endpoint, PrimaryHandlers.RouteValue(c, "endpointId"))),
+ Get("endpointssettings", S.KeyMerge, PrimaryHandlers.EndpointSettings),
+ Patch("endpointssettings/{endpointName?}", S.Broadcast, Broadcast),
+ Get("heartbeats/stats", S.Sum, PrimaryHandlers.SumNumbers),
+
+ // Custom checks and event log
+ Get("customchecks", S.SortedPage, PrimaryHandlers.CustomChecks),
+ Delete("customchecks/{id}", S.RouteToOwner, (f, c) => f.RouteToOwners(c, Surface.Primary, OwnedRecord.CustomCheck, PrimaryHandlers.RouteValue(c, "id"))),
+ Get("eventlogitems", S.SortedPage, PrimaryHandlers.EventLog),
+
+ // Retry redirects
+ Get("redirects", S.SortedPage, PrimaryHandlers.Redirects),
+ Head("redirect", S.Sum, (f, c) => f.SumCount(c, Surface.Primary)),
+ Post("redirects", S.RouteToOwner, PrimaryHandlers.CreateRedirect),
+ Put("redirects/{messageRedirectId}", S.RouteToOwner, (f, c) => f.RouteToOwners(c, Surface.Primary, OwnedRecord.Redirect, PrimaryHandlers.RouteValue(c, "messageRedirectId"))),
+ Delete("redirects/{messageRedirectId}", S.RouteToOwner, (f, c) => f.RouteToOwners(c, Surface.Primary, OwnedRecord.Redirect, PrimaryHandlers.RouteValue(c, "messageRedirectId"))),
+
+ // Notifications
+ Get("notifications/email", S.Lead, (f, c) => f.Lead(c, Surface.Primary)),
+ Post("notifications/email", S.Broadcast, Broadcast),
+ Post("notifications/email/toggle", S.Broadcast, Broadcast),
+ Post("notifications/email/test", S.Broadcast, Broadcast),
+
+ // Retention maintenance
+ Post("maintenance/retention/purge", S.Broadcast, Broadcast),
+ Get("maintenance/retention/purge/status", S.KeyMerge, PrimaryHandlers.RetentionStatus),
+
+ // Licensing component (usage reporting)
+ Get("licensing/endpoints", S.KeyMerge, LicensingHandlers.Endpoints),
+ Post("licensing/endpoints/update", S.Broadcast, Broadcast),
+ Get("licensing/report/available", S.Synthesised, LicensingHandlers.ReportAvailable),
+ Get("licensing/report/file", S.Synthesised, LicensingHandlers.ReportFile),
+ Get("licensing/settings/info", S.Lead, (f, c) => f.Lead(c, Surface.Primary)),
+ Get("licensing/settings/test", S.Synthesised, LicensingHandlers.SettingsTest),
+ Get("licensing/settings/masks", S.Lead, (f, c) => f.Lead(c, Surface.Primary)),
+ Post("licensing/settings/masks/update", S.Broadcast, Broadcast),
+ ];
+
+ static System.Threading.Tasks.Task Broadcast(Federator federator, Microsoft.AspNetCore.Http.HttpContext context) => federator.Broadcast(context, Surface.Primary);
+
+ static FederatedRoute Get(string template, string strategy, RouteHandler handler) => Route(["GET"], template, strategy, handler);
+
+ static FederatedRoute Head(string template, string strategy, RouteHandler handler) => Route(["HEAD"], template, strategy, handler);
+
+ static FederatedRoute Post(string template, string strategy, RouteHandler handler) => Route(["POST"], template, strategy, handler);
+
+ static FederatedRoute Put(string template, string strategy, RouteHandler handler) => Route(["PUT"], template, strategy, handler);
+
+ static FederatedRoute Patch(string template, string strategy, RouteHandler handler) => Route(["PATCH"], template, strategy, handler);
+
+ static FederatedRoute Delete(string template, string strategy, RouteHandler handler) => Route(["DELETE"], template, strategy, handler);
+
+ static FederatedRoute Options(string template, string strategy, RouteHandler handler) => Route(["OPTIONS"], template, strategy, handler);
+
+ static FederatedRoute Route(string[] methods, string template, string strategy, RouteHandler handler) => new(Surface.Primary, methods, template, strategy, handler);
+}
diff --git a/src/ServiceControl.Federation/Configuration/FederationSettings.cs b/src/ServiceControl.Federation/Configuration/FederationSettings.cs
new file mode 100644
index 0000000000..436975382c
--- /dev/null
+++ b/src/ServiceControl.Federation/Configuration/FederationSettings.cs
@@ -0,0 +1,150 @@
+namespace ServiceControl.Federation.Configuration;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+///
+/// Settings for the federated API a primary instance hosts when federation is enabled. Each member is
+/// one independent ServiceControl installation: a primary instance (with its own audit instances behind
+/// it) and, optionally, a Monitoring instance.
+///
+public class FederationSettings
+{
+ public const string DefaultPrimaryPathBase = "/federation/api";
+ public const string DefaultMonitoringPathBase = "/federation/monitoring";
+
+ public string DisplayName { get; set; } = "ServiceControl (federated)";
+
+ ///
+ /// The federated installations. The first member is the lead member: it answers instance-wide
+ /// questions that cannot be aggregated, and its answer wins conflicts.
+ ///
+ public List Members { get; set; } = [];
+
+ public TimeSpan MemberTimeout { get; set; } = TimeSpan.FromSeconds(4);
+
+ ///
+ /// Timeout for streamed transfers such as message bodies and usage report downloads.
+ ///
+ public TimeSpan TransferTimeout { get; set; } = TimeSpan.FromSeconds(60);
+
+ public TimeSpan HealthProbeInterval { get; set; } = TimeSpan.FromSeconds(15);
+
+ ///
+ /// The deepest row a sorted page merge asks each member for. Pages beyond it fall back to the
+ /// approximate per-member page union and say so in X-Particular-Incomplete-Results.
+ ///
+ public int DeepPagingMaxRows { get; set; } = 1000;
+
+ public string PrimaryPathBase { get; set; } = DefaultPrimaryPathBase;
+
+ public string MonitoringPathBase { get; set; } = DefaultMonitoringPathBase;
+
+ public bool HasMonitoring => Members.Any(m => !string.IsNullOrWhiteSpace(m.MonitoringUrl));
+
+ public void Validate()
+ {
+ if (Members.Count == 0)
+ {
+ throw new InvalidOperationException("Federation is enabled but no federated instances are configured. List them in the FederatedInstances setting.");
+ }
+
+ foreach (var member in Members)
+ {
+ if (string.IsNullOrWhiteSpace(member.Key) || !member.Key.All(c => char.IsAsciiLetterOrDigit(c) || (c is '-' or '_')))
+ {
+ throw new InvalidOperationException($"Federated instance key '{member.Key}' is invalid. Use letters, digits, '-' or '_'.");
+ }
+
+ if (!Uri.TryCreate(member.PrimaryUrl, UriKind.Absolute, out _))
+ {
+ throw new InvalidOperationException($"Federated instance '{member.Key}' has an invalid api_uri '{member.PrimaryUrl}'.");
+ }
+
+ if (!string.IsNullOrWhiteSpace(member.MonitoringUrl) && !Uri.TryCreate(member.MonitoringUrl, UriKind.Absolute, out _))
+ {
+ throw new InvalidOperationException($"Federated instance '{member.Key}' has an invalid monitoring_uri '{member.MonitoringUrl}'.");
+ }
+ }
+
+ var duplicate = Members.GroupBy(m => m.Key, StringComparer.OrdinalIgnoreCase).FirstOrDefault(g => g.Count() > 1);
+ if (duplicate != null)
+ {
+ throw new InvalidOperationException($"Federated instance key '{duplicate.Key}' is used more than once. Give the instances distinct 'key' values.");
+ }
+
+ if (DeepPagingMaxRows < 1)
+ {
+ throw new InvalidOperationException("DeepPagingMaxRows must be at least 1.");
+ }
+ }
+
+ ///
+ /// Parses the FederatedInstances setting, a JSON array in the style of RemoteInstances:
+ /// [{"api_uri": "http://host:33333/api/", "monitoring_uri": "http://host:33633/", "key": "eu", "name": "Europe"}].
+ /// Only api_uri is required. A missing key is derived from the host and port of api_uri.
+ ///
+ public static MemberSettings[] ParseMembers(string? json)
+ {
+ if (string.IsNullOrWhiteSpace(json))
+ {
+ return [];
+ }
+
+ var members = JsonSerializer.Deserialize(json) ?? [];
+ foreach (var member in members.Where(m => string.IsNullOrWhiteSpace(m.Key)))
+ {
+ member.Key = KeyFrom(member.PrimaryUrl);
+ }
+
+ return members;
+ }
+
+ ///
+ /// A key that stays the same while the URL does, so ids that embed it remain routable.
+ ///
+ static string KeyFrom(string url)
+ {
+ if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
+ {
+ return "";
+ }
+
+ var key = new StringBuilder();
+ foreach (var c in $"{uri.Host}-{uri.Port}".ToLowerInvariant())
+ {
+ key.Append((char.IsAsciiLetterOrDigit(c) || c == '-') ? c : '-');
+ }
+
+ return key.ToString();
+ }
+}
+
+public class MemberSettings
+{
+ ///
+ /// Short, stable identifier. It appears in synthesised ids such as instance_id, so changing it
+ /// breaks links a client has cached.
+ ///
+ [JsonPropertyName("key")]
+ public string Key { get; set; } = "";
+
+ [JsonPropertyName("name")]
+ public string? Name { get; set; }
+
+ ///
+ /// The primary instance API root, for example http://host:33333/api/.
+ ///
+ [JsonPropertyName("api_uri")]
+ public string PrimaryUrl { get; set; } = "";
+
+ ///
+ /// The Monitoring instance root, for example http://host:33633/. Optional.
+ ///
+ [JsonPropertyName("monitoring_uri")]
+ public string? MonitoringUrl { get; set; }
+}
diff --git a/src/ServiceControl.Federation/FederationHostingExtensions.cs b/src/ServiceControl.Federation/FederationHostingExtensions.cs
new file mode 100644
index 0000000000..d762d6af51
--- /dev/null
+++ b/src/ServiceControl.Federation/FederationHostingExtensions.cs
@@ -0,0 +1,86 @@
+namespace ServiceControl.Federation;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using Api;
+using Configuration;
+using Members;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Routing;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Routing;
+
+///
+/// Registration entry points used by the hosting ServiceControl primary instance, in the same way it
+/// hosts the integrated ServicePulse. The host owns the cross-cutting middleware: forwarded headers,
+/// CORS, authentication and authorization.
+///
+public static class FederationHostingExtensions
+{
+ public static IServiceCollection AddServiceControlFederation(this IServiceCollection services, FederationSettings settings)
+ {
+ settings.Validate();
+
+ services.AddSingleton(settings);
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddHostedService();
+
+ services
+ .AddHttpClient(MemberGateway.HttpClientName, client => client.Timeout = System.Threading.Timeout.InfiniteTimeSpan)
+ .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler
+ {
+ AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli,
+ PooledConnectionLifetime = TimeSpan.FromMinutes(5)
+ });
+
+ return services;
+ }
+
+ ///
+ /// Maps both federated surfaces under their path bases. Routes a table does not list are answered
+ /// by the lead member and logged, so a newer ServiceControl route keeps working rather than failing.
+ ///
+ public static IEndpointRouteBuilder MapServiceControlFederation(this IEndpointRouteBuilder endpoints)
+ {
+ var federator = endpoints.ServiceProvider.GetRequiredService();
+ var settings = federator.Settings;
+
+ MapSurface(endpoints, federator, settings.PrimaryPathBase, PrimaryRoutes.All, Surface.Primary);
+ MapSurface(endpoints, federator, settings.MonitoringPathBase, MonitoringRoutes.All, Surface.Monitoring);
+
+ return endpoints;
+ }
+
+ static void MapSurface(IEndpointRouteBuilder endpoints, Federator federator, string prefix, IEnumerable routes, Surface surface)
+ {
+ var group = endpoints.MapGroup(prefix);
+
+ foreach (var route in routes)
+ {
+ var endpoint = group.MapMethods(route.Template, route.Methods, context => route.Handler(federator, context))
+ .WithDisplayName($"Federation {surface} {string.Join('/', route.Methods)} {route.Template} [{route.Strategy}]");
+
+ // The members decide what the caller may do; anonymous routes on the members stay anonymous here.
+ if (route.Anonymous)
+ {
+ endpoint.AllowAnonymous();
+ }
+ }
+
+ group.Map("{**unmapped}", context =>
+ {
+ federator.Logger.LogWarning("No federation strategy for {Method} {Path}; answering from the lead member", context.Request.Method, context.Request.Path);
+ return federator.Lead(context, surface);
+ });
+ }
+
+ public static IReadOnlyList AllRoutes => [.. PrimaryRoutes.All.Concat(MonitoringRoutes.All)];
+}
diff --git a/src/ServiceControl.Federation/Http/EntityTags.cs b/src/ServiceControl.Federation/Http/EntityTags.cs
new file mode 100644
index 0000000000..f408a21be5
--- /dev/null
+++ b/src/ServiceControl.Federation/Http/EntityTags.cs
@@ -0,0 +1,79 @@
+namespace ServiceControl.Federation.Http;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using Members;
+using Microsoft.Net.Http.Headers;
+
+///
+/// Composes one validator from the members' validators, following the rules in
+/// docs/data-versioning-design.md: absence never produces a tag, and the composite is independent of
+/// the order members answered in.
+///
+public static class EntityTags
+{
+ ///
+ /// A weak tag over every participating member's tag, or null when any of them has none.
+ ///
+ /// The responses that make up the result.
+ /// Anything else the proxy folded into the body, such as a page window.
+ public static string? Combine(IReadOnlyCollection responses, string variant = "")
+ {
+ if (responses.Count == 0 || responses.Any(r => r.Failure != null || (!r.IsNotFound && r.ETag == null)))
+ {
+ return null;
+ }
+
+ using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
+ Append(hash, Generation);
+ Append(hash, variant);
+ foreach (var response in responses.OrderBy(r => r.Member.Key, StringComparer.Ordinal))
+ {
+ Append(hash, response.Member.Key);
+ Append(hash, response.IsNotFound ? "404" : Opaque(response.ETag!));
+ }
+
+ var digest = hash.GetHashAndReset();
+ return $"W/\"{new Guid(digest.AsSpan(0, 16))}\"";
+ }
+
+ ///
+ /// Whether an If-None-Match request header matches the tag, using weak comparison.
+ ///
+ public static bool Matches(string? ifNoneMatch, string? etag)
+ {
+ if (string.IsNullOrEmpty(ifNoneMatch) || etag == null)
+ {
+ return false;
+ }
+
+ if (ifNoneMatch.Trim() == "*")
+ {
+ return true;
+ }
+
+ if (!EntityTagHeaderValue.TryParse(etag, out var current)
+ || !EntityTagHeaderValue.TryParseList([ifNoneMatch], out var candidates))
+ {
+ return false;
+ }
+
+ return candidates.Any(candidate => candidate.Compare(current, useStrongComparison: false));
+ }
+
+ static string Opaque(string etag) => etag.StartsWith("W/", StringComparison.Ordinal) ? etag[2..] : etag;
+
+ static void Append(IncrementalHash hash, string value)
+ {
+ var bytes = Encoding.UTF8.GetBytes(value);
+ hash.AppendData(BitConverter.GetBytes(bytes.Length));
+ hash.AppendData(bytes);
+ }
+
+ // Bumped whenever the proxy changes how it shapes a response, so cached pages from an older proxy
+ // are not served as current.
+ const string Generation = "federation-1";
+}
diff --git a/src/ServiceControl.Federation/Http/ProxyRequest.cs b/src/ServiceControl.Federation/Http/ProxyRequest.cs
new file mode 100644
index 0000000000..eea063c274
--- /dev/null
+++ b/src/ServiceControl.Federation/Http/ProxyRequest.cs
@@ -0,0 +1,125 @@
+namespace ServiceControl.Federation.Http;
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Members;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Extensions;
+using Microsoft.AspNetCore.Http.Features;
+using Microsoft.Extensions.Primitives;
+
+///
+/// Reads what the proxy needs from an incoming request: the member-relative path, the caller's
+/// credentials and body, and the paging parameters.
+///
+public static class ProxyRequest
+{
+ ///
+ /// The request target relative to the surface prefix, exactly as the client encoded it, for
+ /// example recoverability/groups/Endpoint%20Name?classifierFilter=Sales.
+ ///
+ public static string RelativeTarget(HttpContext context, string surfacePrefix)
+ {
+ var raw = context.Features.Get()?.RawTarget;
+ var prefix = context.Request.PathBase.Add(new PathString(surfacePrefix)).ToUriComponent();
+
+ string relative;
+ if (!string.IsNullOrEmpty(raw) && raw.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
+ {
+ relative = raw[prefix.Length..];
+ }
+ else
+ {
+ var path = context.Request.Path.ToUriComponent();
+ relative = (path.StartsWith(surfacePrefix, StringComparison.OrdinalIgnoreCase) ? path[surfacePrefix.Length..] : path)
+ + context.Request.QueryString.ToUriComponent();
+ }
+
+ // Clients sometimes build "api//messages/..." (see BodyUrlRouteFix in ServiceControl).
+ return relative.TrimStart('/');
+ }
+
+ public static string RelativePath(HttpContext context, string surfacePrefix)
+ {
+ var target = RelativeTarget(context, surfacePrefix);
+ var queryStart = target.IndexOf('?');
+ return queryStart < 0 ? target : target[..queryStart];
+ }
+
+ ///
+ /// The same relative path with a replaced query string.
+ ///
+ public static string WithQuery(HttpContext context, string surfacePrefix, IEnumerable> query)
+ {
+ var builder = new QueryBuilder(query);
+ return RelativePath(context, surfacePrefix) + builder.ToQueryString().ToUriComponent();
+ }
+
+ ///
+ /// The incoming query with some parameters replaced or added.
+ ///
+ public static List> QueryWith(HttpContext context, params (string Name, string Value)[] overrides)
+ {
+ var names = overrides.Select(o => o.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
+ var query = context.Request.Query
+ .Where(q => !names.Contains(q.Key))
+ .SelectMany(q => q.Value.Select(v => new KeyValuePair(q.Key, v ?? "")))
+ .ToList();
+ query.AddRange(overrides.Select(o => new KeyValuePair(o.Name, o.Value)));
+ return query;
+ }
+
+ public static string? Authorization(HttpContext context) =>
+ context.Request.Headers.Authorization is { Count: > 0 } value ? value.ToString() : null;
+
+ public static async Task ReadBody(HttpContext context, CancellationToken cancellationToken = default)
+ {
+ if (context.Request.ContentLength == 0 || (context.Request.ContentLength == null && !context.Request.Headers.ContainsKey("Transfer-Encoding")))
+ {
+ return null;
+ }
+
+ using var buffer = new MemoryStream();
+ await context.Request.Body.CopyToAsync(buffer, cancellationToken);
+ return buffer.ToArray();
+ }
+
+ ///
+ /// A member request that repeats the incoming one: same method, target, credentials and body.
+ ///
+ public static async Task Repeat(HttpContext context, string surfacePrefix, CancellationToken cancellationToken = default) =>
+ new(
+ new HttpMethod(context.Request.Method),
+ RelativeTarget(context, surfacePrefix),
+ Authorization(context),
+ await ReadBody(context, cancellationToken),
+ context.Request.ContentType,
+ context.Request.Headers.Accept is { Count: > 0 } accept ? accept.ToString() : null);
+
+ public static MemberRequest Get(HttpContext context, string pathAndQuery) =>
+ new(HttpMethod.Get, pathAndQuery, Authorization(context));
+
+ ///
+ /// The proxy's own base URL for a surface, as the client sees it (forwarded headers applied).
+ ///
+ public static string BaseUrl(HttpContext context, string surfacePrefix)
+ {
+ var request = context.Request;
+ return $"{request.Scheme}://{request.Host}{request.PathBase}{surfacePrefix.TrimEnd('/')}/";
+ }
+
+ public static int IntQuery(HttpContext context, string name, int defaultValue)
+ {
+ var value = context.Request.Query[name];
+ return int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) && parsed > 0 ? parsed : defaultValue;
+ }
+
+ public static string? Query(HttpContext context, string name) =>
+ context.Request.Query.TryGetValue(name, out var value) && !StringValues.IsNullOrEmpty(value) ? value.ToString() : null;
+}
diff --git a/src/ServiceControl.Federation/Http/ProxyResponse.cs b/src/ServiceControl.Federation/Http/ProxyResponse.cs
new file mode 100644
index 0000000000..29dcf09a9e
--- /dev/null
+++ b/src/ServiceControl.Federation/Http/ProxyResponse.cs
@@ -0,0 +1,225 @@
+namespace ServiceControl.Federation.Http;
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Net.Http;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Threading;
+using System.Threading.Tasks;
+using Members;
+using Microsoft.AspNetCore.Http;
+
+///
+/// What the proxy adds to an aggregated response besides the body.
+///
+public sealed class ResponseMeta
+{
+ public int Status { get; init; } = StatusCodes.Status200OK;
+
+ public long? TotalCount { get; init; }
+
+ public string? ETag { get; init; }
+
+ public string? Link { get; init; }
+
+ public string? Version { get; init; }
+
+ public IReadOnlyList Incomplete { get; init; } = [];
+}
+
+public static class ProxyResponse
+{
+ public const string IncompleteResultsHeader = "X-Particular-Incomplete-Results";
+
+ ///
+ /// X-Particular-Incomplete-Results entries: one per member that contributed nothing, named by
+ /// its API URL, plus the entries members reported for their own audit instances.
+ ///
+ public static List IncompleteEntries(IEnumerable responses, Surface surface)
+ {
+ var entries = new List();
+ foreach (var response in responses)
+ {
+ var url = response.Member.BaseUri(surface)?.ToString().TrimEnd('/') ?? response.Member.Key;
+ if (response.Failure is { } reason)
+ {
+ entries.Add($"{url}:{ReasonName(reason)}");
+ }
+ else if (!response.IsSuccess && !response.IsNotFound)
+ {
+ // Answered, but with an error, so it contributed nothing.
+ entries.Add($"{url}:{ReasonName(FailureReason.Error)}");
+ }
+ else if (response.Header(IncompleteResultsHeader) is { Length: > 0 } memberEntries)
+ {
+ entries.AddRange(memberEntries.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries));
+ }
+ }
+
+ return entries;
+ }
+
+ public static string ReasonName(FailureReason reason) => reason switch
+ {
+ FailureReason.Timeout => "timeout",
+ FailureReason.Unavailable => "unavailable",
+ FailureReason.Error => "error",
+ _ => "error"
+ };
+
+ public static async Task WriteJson(HttpContext context, JsonNode? body, ResponseMeta meta, CancellationToken cancellationToken = default)
+ {
+ WriteHeaders(context, meta);
+
+ if (meta.ETag != null && EntityTags.Matches(context.Request.Headers.IfNoneMatch, meta.ETag))
+ {
+ context.Response.StatusCode = StatusCodes.Status304NotModified;
+ return;
+ }
+
+ context.Response.StatusCode = meta.Status;
+ if (HttpMethods.IsHead(context.Request.Method) || body == null)
+ {
+ return;
+ }
+
+ context.Response.ContentType = "application/json; charset=utf-8";
+ await using var writer = new Utf8JsonWriter(context.Response.Body);
+ body.WriteTo(writer);
+ await writer.FlushAsync(cancellationToken);
+ }
+
+ public static void WriteHeaders(HttpContext context, ResponseMeta meta)
+ {
+ var headers = context.Response.Headers;
+ if (meta.TotalCount is { } total)
+ {
+ headers["Total-Count"] = total.ToString(CultureInfo.InvariantCulture);
+ }
+
+ if (meta.ETag != null)
+ {
+ headers.ETag = meta.ETag;
+ }
+
+ if (meta.Link != null)
+ {
+ headers.Link = meta.Link;
+ }
+
+ if (meta.Version != null)
+ {
+ headers["X-Particular-Version"] = meta.Version;
+ }
+
+ if (meta.Incomplete.Count > 0)
+ {
+ headers[IncompleteResultsHeader] = string.Join(", ", meta.Incomplete);
+ }
+
+ headers.CacheControl = "private, max-age=0";
+ headers.Vary = "Accept";
+ }
+
+ ///
+ /// Relays a buffered member response unchanged, apart from headers that describe the hop.
+ ///
+ public static async Task Relay(HttpContext context, MemberResponse response, CancellationToken cancellationToken = default)
+ {
+ context.Response.StatusCode = response.Status;
+ foreach (var (name, value) in response.Headers)
+ {
+ if (!IsHopHeader(name))
+ {
+ context.Response.Headers[name] = value;
+ }
+ }
+
+ if (response.Body.Length > 0 && !HttpMethods.IsHead(context.Request.Method))
+ {
+ await context.Response.Body.WriteAsync(response.Body, cancellationToken);
+ }
+ }
+
+ ///
+ /// Streams a member response to the caller.
+ ///
+ public static async Task RelayStreaming(HttpContext context, HttpResponseMessage response, CancellationToken cancellationToken = default)
+ {
+ context.Response.StatusCode = (int)response.StatusCode;
+ foreach (var header in response.Headers.Concat(response.Content.Headers))
+ {
+ if (!IsHopHeader(header.Key))
+ {
+ context.Response.Headers[header.Key] = header.Value.ToArray();
+ }
+ }
+
+ if (!HttpMethods.IsHead(context.Request.Method))
+ {
+ await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
+ await stream.CopyToAsync(context.Response.Body, cancellationToken);
+ }
+ }
+
+ public static Task Problem(HttpContext context, int status, string title, IReadOnlyList? incomplete = null, CancellationToken cancellationToken = default)
+ {
+ var body = new JsonObject
+ {
+ ["type"] = "https://tools.ietf.org/html/rfc9110#section-15",
+ ["title"] = title,
+ ["status"] = status
+ };
+
+ WriteHeaders(context, new ResponseMeta { Incomplete = incomplete ?? [] });
+ context.Response.StatusCode = status;
+ context.Response.ContentType = "application/problem+json";
+ return context.Response.WriteAsync(body.ToJsonString(), cancellationToken);
+ }
+
+ ///
+ /// Paging links in the shape ServiceControl emits: relative to the API root, with the page replaced.
+ ///
+ public static string? PagingLinks(HttpContext context, string surfacePrefix, int page, int pageSize, long total)
+ {
+ if (total <= pageSize)
+ {
+ return null;
+ }
+
+ var lastPage = (int)Math.Ceiling(total / (double)pageSize);
+ var links = new List();
+
+ void Add(int target, string rel)
+ {
+ var query = ProxyRequest.QueryWith(context, ("page", target.ToString(CultureInfo.InvariantCulture)), ("per_page", pageSize.ToString(CultureInfo.InvariantCulture)));
+ links.Add($"<{ProxyRequest.WithQuery(context, surfacePrefix, query)}>; rel=\"{rel}\"");
+ }
+
+ if (page < lastPage)
+ {
+ Add(page + 1, "next");
+ }
+
+ if (page > 1)
+ {
+ Add(page - 1, "prev");
+ }
+
+ Add(1, "first");
+ Add(lastPage, "last");
+ return string.Join(", ", links);
+ }
+
+ static bool IsHopHeader(string name) =>
+ HopHeaders.Contains(name) || name.StartsWith("Access-Control-", StringComparison.OrdinalIgnoreCase);
+
+ static readonly HashSet HopHeaders = new(StringComparer.OrdinalIgnoreCase)
+ {
+ "Connection", "Keep-Alive", "Transfer-Encoding", "Upgrade", "Proxy-Connection", "Content-Length",
+ "Content-Encoding", "Server", "Date", "Vary"
+ };
+}
diff --git a/src/ServiceControl.Federation/Members/Member.cs b/src/ServiceControl.Federation/Members/Member.cs
new file mode 100644
index 0000000000..ec8e4f27cd
--- /dev/null
+++ b/src/ServiceControl.Federation/Members/Member.cs
@@ -0,0 +1,94 @@
+namespace ServiceControl.Federation.Members;
+
+using System;
+using Configuration;
+
+///
+/// Which of a member's two HTTP surfaces a request targets.
+///
+public enum Surface
+{
+ Primary,
+ Monitoring
+}
+
+///
+/// Runtime view of one configured installation, including its last observed health and version.
+///
+public sealed class Member : IEquatable
+{
+ public Member(MemberSettings settings, bool isLead)
+ {
+ Key = settings.Key;
+ Name = string.IsNullOrWhiteSpace(settings.Name) ? settings.Key : settings.Name;
+ PrimaryUri = WithTrailingSlash(settings.PrimaryUrl);
+ MonitoringUri = string.IsNullOrWhiteSpace(settings.MonitoringUrl) ? null : WithTrailingSlash(settings.MonitoringUrl);
+ IsLead = isLead;
+ }
+
+ public string Key { get; }
+
+ public string Name { get; }
+
+ public Uri PrimaryUri { get; }
+
+ public Uri? MonitoringUri { get; }
+
+ public bool IsLead { get; }
+
+ public Uri? BaseUri(Surface surface) => surface == Surface.Primary ? PrimaryUri : MonitoringUri;
+
+ public bool Serves(Surface surface) => BaseUri(surface) != null;
+
+ ///
+ /// Set when a connection attempt failed. Requests skip the member until the health probe succeeds,
+ /// so a dead member does not cost a timeout on every poll.
+ ///
+ public bool IsUnavailable(Surface surface) => surface == Surface.Primary ? primaryUnavailable : monitoringUnavailable;
+
+ public void MarkUnavailable(Surface surface, bool unavailable)
+ {
+ if (surface == Surface.Primary)
+ {
+ primaryUnavailable = unavailable;
+ }
+ else
+ {
+ monitoringUnavailable = unavailable;
+ }
+ }
+
+ public string? Version(Surface surface) => surface == Surface.Primary ? primaryVersion : monitoringVersion;
+
+ public void RecordVersion(Surface surface, string? version)
+ {
+ if (string.IsNullOrWhiteSpace(version))
+ {
+ return;
+ }
+
+ if (surface == Surface.Primary)
+ {
+ primaryVersion = version;
+ }
+ else
+ {
+ monitoringVersion = version;
+ }
+ }
+
+ public override string ToString() => Key;
+
+ public bool Equals(Member? other) => other != null && string.Equals(Key, other.Key, StringComparison.OrdinalIgnoreCase);
+
+ public override bool Equals(object? obj) => Equals(obj as Member);
+
+ public override int GetHashCode() => StringComparer.OrdinalIgnoreCase.GetHashCode(Key);
+
+ static Uri WithTrailingSlash(string url) => new(url.EndsWith('/') ? url : url + "/");
+
+ volatile bool primaryUnavailable;
+ volatile bool monitoringUnavailable;
+ volatile string? primaryVersion;
+ volatile string? monitoringVersion;
+}
diff --git a/src/ServiceControl.Federation/Members/MemberGateway.cs b/src/ServiceControl.Federation/Members/MemberGateway.cs
new file mode 100644
index 0000000000..ccabf39914
--- /dev/null
+++ b/src/ServiceControl.Federation/Members/MemberGateway.cs
@@ -0,0 +1,209 @@
+namespace ServiceControl.Federation.Members;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Threading;
+using System.Threading.Tasks;
+using Configuration;
+using Microsoft.Extensions.Logging;
+
+///
+/// Sends requests to members and classifies the outcome the same way ServiceControl's own
+/// scatter-gather does: connection failures mark the member unavailable, timeouts and 504s are
+/// timeouts, and every other non-success status is passed back for the caller to judge.
+///
+public sealed class MemberGateway(IHttpClientFactory httpClientFactory, FederationSettings settings, ILogger logger)
+{
+ public const string HttpClientName = "federation-members";
+
+ public Task SendAll(IEnumerable members, Surface surface, MemberRequest request, CancellationToken cancellationToken = default) =>
+ Task.WhenAll(members.Select(member => Send(member, surface, request, cancellationToken)));
+
+ public async Task Send(Member member, Surface surface, MemberRequest request, CancellationToken cancellationToken = default)
+ {
+ var baseUri = member.BaseUri(surface);
+ if (baseUri == null)
+ {
+ return MemberResponse.Failed(member, FailureReason.Unavailable);
+ }
+
+ if (member.IsUnavailable(surface))
+ {
+ return MemberResponse.Failed(member, FailureReason.Unavailable);
+ }
+
+ using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ timeout.CancelAfter(settings.MemberTimeout);
+
+ try
+ {
+ using var message = CreateMessage(baseUri, request);
+ using var response = await httpClientFactory.CreateClient(HttpClientName).SendAsync(message, timeout.Token);
+ var body = await response.Content.ReadAsByteArrayAsync(timeout.Token);
+ var headers = CollectHeaders(response);
+ member.RecordVersion(surface, response.Headers.TryGetValues("X-Particular-Version", out var versions) ? versions.FirstOrDefault() : null);
+
+ var status = (int)response.StatusCode;
+ FailureReason? failure = status == 504 ? FailureReason.Timeout : null;
+ if (status is >= 500 and not 504)
+ {
+ failure = FailureReason.Error;
+ }
+
+ if (failure != null)
+ {
+ logger.LogWarning("Member {Member} answered {Status} for {Method} {Path}", member.Key, status, request.Method, request.PathAndQuery);
+ }
+
+ return new MemberResponse(member, status, headers, body, failure);
+ }
+ catch (HttpRequestException ex)
+ {
+ logger.LogWarning(ex, "Member {Member} is unreachable at {BaseUri}; skipping it until the health probe succeeds", member.Key, baseUri);
+ member.MarkUnavailable(surface, true);
+ return MemberResponse.Failed(member, FailureReason.Unavailable);
+ }
+ catch (OperationCanceledException) when (timeout.Token.IsCancellationRequested)
+ {
+ if (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+
+ // Intentional: the per-member timeout fired
+ logger.LogWarning("Member {Member} timed out after {Timeout} for {Method} {Path}", member.Key, settings.MemberTimeout, request.Method, request.PathAndQuery);
+ return MemberResponse.Failed(member, FailureReason.Timeout);
+ }
+ }
+
+ ///
+ /// Sends a request whose response is streamed to the caller rather than buffered. The caller owns
+ /// the returned response. Returns null when the member could not be reached.
+ ///
+ public async Task SendStreaming(Member member, Surface surface, MemberRequest request, CancellationToken cancellationToken = default)
+ {
+ var baseUri = member.BaseUri(surface);
+ if (baseUri == null || member.IsUnavailable(surface))
+ {
+ return null;
+ }
+
+ using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ timeout.CancelAfter(settings.TransferTimeout);
+
+ try
+ {
+ using var message = CreateMessage(baseUri, request);
+ return await httpClientFactory.CreateClient(HttpClientName).SendAsync(message, HttpCompletionOption.ResponseHeadersRead, timeout.Token);
+ }
+ catch (HttpRequestException ex)
+ {
+ logger.LogWarning(ex, "Member {Member} is unreachable at {BaseUri}", member.Key, baseUri);
+ member.MarkUnavailable(surface, true);
+ return null;
+ }
+ catch (OperationCanceledException) when (timeout.Token.IsCancellationRequested)
+ {
+ if (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+
+ // Intentional: the transfer timeout fired
+ logger.LogWarning("Member {Member} timed out streaming {Method} {Path}", member.Key, request.Method, request.PathAndQuery);
+ return null;
+ }
+ }
+
+ ///
+ /// Health probe: bypasses the unavailable flag and resets it on success.
+ ///
+ public async Task Probe(Member member, Surface surface, CancellationToken cancellationToken = default)
+ {
+ var baseUri = member.BaseUri(surface);
+ if (baseUri == null)
+ {
+ return false;
+ }
+
+ using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ timeout.CancelAfter(settings.MemberTimeout);
+
+ try
+ {
+ using var message = new HttpRequestMessage(HttpMethod.Get, baseUri);
+ using var response = await httpClientFactory.CreateClient(HttpClientName).SendAsync(message, timeout.Token);
+ member.RecordVersion(surface, response.Headers.TryGetValues("X-Particular-Version", out var versions) ? versions.FirstOrDefault() : null);
+ // Any HTTP answer means the member is reachable. Requests then classify their own responses;
+ // only connection failures and timeouts take a member out of rotation.
+ if (member.IsUnavailable(surface))
+ {
+ logger.LogInformation("Member {Member} {Surface} is reachable again", member.Key, surface);
+ }
+
+ member.MarkUnavailable(surface, false);
+ return true;
+ }
+ catch (OperationCanceledException) when (timeout.Token.IsCancellationRequested)
+ {
+ if (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+
+ // Intentional: the probe timed out
+ return Unreachable(member, surface, baseUri);
+ }
+ catch (HttpRequestException)
+ {
+ return Unreachable(member, surface, baseUri);
+ }
+ }
+
+ bool Unreachable(Member member, Surface surface, Uri baseUri)
+ {
+ if (!member.IsUnavailable(surface))
+ {
+ logger.LogWarning("Member {Member} {Surface} at {BaseUri} is unreachable", member.Key, surface, baseUri);
+ }
+
+ member.MarkUnavailable(surface, true);
+ return false;
+ }
+
+ static HttpRequestMessage CreateMessage(Uri baseUri, MemberRequest request)
+ {
+ var message = new HttpRequestMessage(request.Method, new Uri(baseUri, request.PathAndQuery));
+ message.Headers.Accept.ParseAdd(request.Accept ?? "application/json");
+
+ if (!string.IsNullOrEmpty(request.Authorization))
+ {
+ message.Headers.TryAddWithoutValidation("Authorization", request.Authorization);
+ }
+
+ if (request.Body != null)
+ {
+ message.Content = new ByteArrayContent(request.Body);
+ if (!string.IsNullOrEmpty(request.ContentType))
+ {
+ message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(request.ContentType);
+ }
+ }
+
+ return message;
+ }
+
+ static Dictionary CollectHeaders(HttpResponseMessage response)
+ {
+ var headers = new Dictionary(StringComparer.OrdinalIgnoreCase);
+ foreach (var header in response.Headers.Concat(response.Content.Headers))
+ {
+ headers[header.Key] = string.Join(", ", header.Value);
+ }
+
+ return headers;
+ }
+}
diff --git a/src/ServiceControl.Federation/Members/MemberHealthMonitor.cs b/src/ServiceControl.Federation/Members/MemberHealthMonitor.cs
new file mode 100644
index 0000000000..39262ca69d
--- /dev/null
+++ b/src/ServiceControl.Federation/Members/MemberHealthMonitor.cs
@@ -0,0 +1,46 @@
+namespace ServiceControl.Federation.Members;
+
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Configuration;
+using Microsoft.Extensions.Hosting;
+
+///
+/// Probes every member's root document on an interval, which re-enables members marked unavailable
+/// and keeps the last known versions current for the version header.
+///
+sealed class MemberHealthMonitor(MemberRegistry registry, MemberGateway gateway, FederationSettings settings) : BackgroundService
+{
+ protected override async Task ExecuteAsync(CancellationToken cancellationToken = default)
+ {
+ using var timer = new PeriodicTimer(settings.HealthProbeInterval);
+ try
+ {
+ do
+ {
+ var probes = registry.All
+ .SelectMany(member => new[] { Surface.Primary, Surface.Monitoring }.Where(member.Serves).Select(surface => gateway.Probe(member, surface, cancellationToken)));
+ await Task.WhenAll(probes);
+ }
+ while (await WaitForNextTick(timer, cancellationToken));
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ // Host is stopping
+ }
+ }
+
+ static async Task WaitForNextTick(PeriodicTimer timer, CancellationToken cancellationToken)
+ {
+ try
+ {
+ return await timer.WaitForNextTickAsync(cancellationToken);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ return false;
+ }
+ }
+}
diff --git a/src/ServiceControl.Federation/Members/MemberRegistry.cs b/src/ServiceControl.Federation/Members/MemberRegistry.cs
new file mode 100644
index 0000000000..54ff6a656b
--- /dev/null
+++ b/src/ServiceControl.Federation/Members/MemberRegistry.cs
@@ -0,0 +1,21 @@
+namespace ServiceControl.Federation.Members;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Configuration;
+
+///
+/// The configured members in configuration order; the first is the lead member. Every fan-out iterates
+/// members in this order, so wherever a merge keeps "the first" value, the lead member wins.
+///
+public sealed class MemberRegistry(FederationSettings settings)
+{
+ public IReadOnlyList All { get; } = [.. settings.Members.Select((m, index) => new Member(m, isLead: index == 0))];
+
+ public Member Lead => All[0];
+
+ public IReadOnlyList Serving(Surface surface) => [.. All.Where(m => m.Serves(surface))];
+
+ public Member? Find(string key) => All.FirstOrDefault(m => string.Equals(m.Key, key, StringComparison.OrdinalIgnoreCase));
+}
diff --git a/src/ServiceControl.Federation/Members/MemberResponse.cs b/src/ServiceControl.Federation/Members/MemberResponse.cs
new file mode 100644
index 0000000000..a4223d57b1
--- /dev/null
+++ b/src/ServiceControl.Federation/Members/MemberResponse.cs
@@ -0,0 +1,94 @@
+namespace ServiceControl.Federation.Members;
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Net.Http;
+using System.Text.Json.Nodes;
+
+///
+/// Why a member contributed nothing to an aggregated answer. The names are the reasons ServicePulse
+/// understands in X-Particular-Incomplete-Results.
+///
+public enum FailureReason
+{
+ Timeout,
+ Unavailable,
+ Error
+}
+
+public sealed record MemberRequest(
+ HttpMethod Method,
+ string PathAndQuery,
+ string? Authorization,
+ byte[]? Body = null,
+ string? ContentType = null,
+ string? Accept = null);
+
+///
+/// A buffered member response. Bodies of aggregated calls are small JSON documents, so they are read in
+/// full; streamed transfers do not go through this type.
+///
+public sealed class MemberResponse
+{
+ public MemberResponse(Member member, int status, IReadOnlyDictionary headers, byte[] body, FailureReason? failure)
+ {
+ Member = member;
+ Status = status;
+ Headers = headers;
+ Body = body;
+ Failure = failure;
+ }
+
+ public static MemberResponse Failed(Member member, FailureReason reason) =>
+ new(member, 0, new Dictionary(StringComparer.OrdinalIgnoreCase), [], reason);
+
+ public Member Member { get; }
+
+ ///
+ /// 0 when no HTTP response was received.
+ ///
+ public int Status { get; }
+
+ public IReadOnlyDictionary Headers { get; }
+
+ public byte[] Body { get; }
+
+ public FailureReason? Failure { get; }
+
+ public bool IsSuccess => Failure == null && Status is >= 200 and < 300;
+
+ public bool IsNotFound => Status == 404;
+
+ public bool IsAuthFailure => Status is 401 or 403;
+
+ public string? Header(string name) => Headers.TryGetValue(name, out var value) ? value : null;
+
+ public string? ContentType => Header("Content-Type");
+
+ public string? ETag => Header("ETag");
+
+ public string? Version => Header("X-Particular-Version");
+
+ public int? TotalCount =>
+ int.TryParse(Header("Total-Count"), NumberStyles.Integer, CultureInfo.InvariantCulture, out var count) ? count : null;
+
+ public JsonNode? Json()
+ {
+ if (!jsonParsed)
+ {
+ json = Body.Length == 0 ? null : JsonNode.Parse(Body);
+ jsonParsed = true;
+ }
+
+ return json;
+ }
+
+ ///
+ /// The parsed body as an array, or an empty array when the member answered 404 or with no body.
+ ///
+ public JsonArray Array() => Json() as JsonArray ?? [];
+
+ JsonNode? json;
+ bool jsonParsed;
+}
diff --git a/src/ServiceControl.Federation/Routing/InstanceIdCodec.cs b/src/ServiceControl.Federation/Routing/InstanceIdCodec.cs
new file mode 100644
index 0000000000..602790fd7e
--- /dev/null
+++ b/src/ServiceControl.Federation/Routing/InstanceIdCodec.cs
@@ -0,0 +1,100 @@
+namespace ServiceControl.Federation.Routing;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json.Nodes;
+
+///
+/// Makes instance_id values unique across members. A member's instance id is the base64url of
+/// a configured URL, which two members can share, so the proxy prefixes it with the member key:
+/// {memberKey}~{memberInstanceId}. An empty member part means the member's own local store.
+///
+///
+/// '~' is not in the base64url alphabet ServiceControl uses (which encodes padding as '.'), so splitting
+/// on the first '~' is unambiguous.
+///
+public static class InstanceIdCodec
+{
+ public const char Separator = '~';
+
+ public static string Encode(string memberKey, string? memberInstanceId) => $"{memberKey}{Separator}{memberInstanceId}";
+
+ public static bool TryDecode(string? value, out string memberKey, out string? memberInstanceId)
+ {
+ memberKey = "";
+ memberInstanceId = null;
+ if (string.IsNullOrEmpty(value))
+ {
+ return false;
+ }
+
+ var separator = value.IndexOf(Separator);
+ if (separator <= 0)
+ {
+ return false;
+ }
+
+ memberKey = value[..separator];
+ var rest = value[(separator + 1)..];
+ memberInstanceId = rest.Length == 0 ? null : rest;
+ return true;
+ }
+
+ ///
+ /// Rewrites every instance_id property and the instance_id query parameter of every
+ /// body_url in a member's response, so that a later body or retry request can be routed back.
+ ///
+ public static void RewriteTree(JsonNode? node, string memberKey)
+ {
+ switch (node)
+ {
+ case JsonObject obj:
+ foreach (var property in obj.ToList())
+ {
+ if (property.Key == "instance_id" && property.Value is JsonValue idValue && idValue.TryGetValue(out var id))
+ {
+ obj[property.Key] = Encode(memberKey, id);
+ }
+ else if (property.Key == "body_url" && property.Value is JsonValue urlValue && urlValue.TryGetValue(out var url))
+ {
+ obj[property.Key] = RewriteBodyUrl(url, memberKey);
+ }
+ else
+ {
+ RewriteTree(property.Value, memberKey);
+ }
+ }
+
+ break;
+ case JsonArray array:
+ foreach (var item in array)
+ {
+ RewriteTree(item, memberKey);
+ }
+
+ break;
+ default:
+ break;
+ }
+ }
+
+ public static string RewriteBodyUrl(string url, string memberKey)
+ {
+ var queryStart = url.IndexOf('?');
+ var path = queryStart < 0 ? url : url[..queryStart];
+ var query = queryStart < 0 ? "" : url[(queryStart + 1)..];
+
+ var parameters = query.Split('&', StringSplitOptions.RemoveEmptyEntries).ToList();
+ var instanceIndex = parameters.FindIndex(p => p.StartsWith("instance_id=", StringComparison.Ordinal));
+ string? memberInstanceId = null;
+ if (instanceIndex >= 0)
+ {
+ memberInstanceId = Uri.UnescapeDataString(parameters[instanceIndex]["instance_id=".Length..]);
+ parameters.RemoveAt(instanceIndex);
+ }
+
+ var rewritten = new List(parameters) { "instance_id=" + Uri.EscapeDataString(Encode(memberKey, memberInstanceId)) };
+ return $"{path}?{string.Join('&', rewritten)}";
+ }
+}
diff --git a/src/ServiceControl.Federation/Routing/OwnerCache.cs b/src/ServiceControl.Federation/Routing/OwnerCache.cs
new file mode 100644
index 0000000000..164e3f77d2
--- /dev/null
+++ b/src/ServiceControl.Federation/Routing/OwnerCache.cs
@@ -0,0 +1,57 @@
+namespace ServiceControl.Federation.Routing;
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+
+///
+/// The kinds of records the proxy routes commands for by id.
+///
+public enum OwnedRecord
+{
+ FailedMessage,
+ CustomCheck,
+ Endpoint,
+ Redirect
+}
+
+///
+/// Remembers which members returned a record, learned from the list and detail responses the proxy
+/// relays. ServicePulse always lists a record before acting on it, so this answers nearly every
+/// command. It is only an optimisation: a miss falls back to asking the members.
+///
+public sealed class OwnerCache
+{
+ public void Record(OwnedRecord kind, string? id, string memberKey)
+ {
+ if (string.IsNullOrEmpty(id))
+ {
+ return;
+ }
+
+ if (owners.Count >= MaxEntries)
+ {
+ owners.Clear();
+ }
+
+ owners.AddOrUpdate((kind, id), _ => [memberKey], (_, existing) => existing.Contains(memberKey) ? existing : existing.Add(memberKey));
+ }
+
+ public IReadOnlySet? Find(OwnedRecord kind, string id) => owners.TryGetValue((kind, id), out var members) ? members : null;
+
+ public void Forget(OwnedRecord kind, string id) => owners.TryRemove((kind, id), out _);
+
+ readonly ConcurrentDictionary<(OwnedRecord Kind, string Id), ImmutableHashSet> owners = new(new KeyComparer());
+
+ const int MaxEntries = 200_000;
+
+ sealed class KeyComparer : IEqualityComparer<(OwnedRecord Kind, string Id)>
+ {
+ public bool Equals((OwnedRecord Kind, string Id) x, (OwnedRecord Kind, string Id) y) =>
+ x.Kind == y.Kind && string.Equals(x.Id, y.Id, StringComparison.OrdinalIgnoreCase);
+
+ public int GetHashCode((OwnedRecord Kind, string Id) obj) =>
+ HashCode.Combine(obj.Kind, StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Id));
+ }
+}
diff --git a/src/ServiceControl.Federation/ServiceControl.Federation.csproj b/src/ServiceControl.Federation/ServiceControl.Federation.csproj
new file mode 100644
index 0000000000..b7cb95d185
--- /dev/null
+++ b/src/ServiceControl.Federation/ServiceControl.Federation.csproj
@@ -0,0 +1,16 @@
+
+
+
+ net10.0
+ enable
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt
index abd4c98313..1794978306 100644
--- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt
+++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt
@@ -45,6 +45,8 @@
"AllowMessageEditing": false,
"EnableIntegratedServicePulse": false,
"ServicePulseSettings": null,
+ "EnableFederation": false,
+ "FederatedInstances": [],
"MessageFilter": null,
"EmailDropFolder": null,
"ValidateConfiguration": true,
diff --git a/src/ServiceControl.slnx b/src/ServiceControl.slnx
index e9a507b944..2f5a52a9f2 100644
--- a/src/ServiceControl.slnx
+++ b/src/ServiceControl.slnx
@@ -61,6 +61,12 @@
+
+
+
+
+
+
diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs
index 6b3ac7c6e0..4506ad9291 100644
--- a/src/ServiceControl/HostApplicationBuilderExtensions.cs
+++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs
@@ -211,6 +211,7 @@ Audit Retention Period (optional): {settings.AuditRetentionPeriod}
ServiceControl Logging Level: {settings.LoggingSettings.LogLevel}
Selected Transport Customization: {settings.TransportType}
Integrated ServicePulse: {(settings.EnableIntegratedServicePulse ? "Enabled" : "Disabled")}
+Federation: {(settings.EnableFederation ? $"Enabled ({settings.FederatedInstances.Length} instances)" : "Disabled")}
-------------------------------------------------------------";
var logger = LoggerUtil.CreateStaticLogger(typeof(HostApplicationBuilderExtensions), settings.LoggingSettings.LogLevel);
diff --git a/src/ServiceControl/Hosting/Commands/RunCommand.cs b/src/ServiceControl/Hosting/Commands/RunCommand.cs
index 33a6361511..716f399392 100644
--- a/src/ServiceControl/Hosting/Commands/RunCommand.cs
+++ b/src/ServiceControl/Hosting/Commands/RunCommand.cs
@@ -9,6 +9,7 @@
using Particular.ServiceControl.Hosting;
using ServiceBus.Management.Infrastructure.Settings;
using ServiceControl;
+ using ServiceControl.Federation;
using ServiceControl.Hosting.Auth;
using ServiceControl.Hosting.Https;
using ServicePulse;
@@ -30,9 +31,17 @@ public override async Task Execute(HostArguments args, Settings settings, Cancel
hostBuilder.AddServiceControlHttps(settings.HttpsSettings);
hostBuilder.AddServiceControl(settings, endpointConfiguration);
hostBuilder.AddServiceControlApi(settings.CorsSettings);
+ if (settings.EnableFederation)
+ {
+ hostBuilder.Services.AddServiceControlFederation(settings.CreateFederationSettings());
+ }
var app = hostBuilder.Build();
app.UseServiceControl(settings.ForwardedHeadersSettings, settings.HttpsSettings);
+ if (settings.EnableFederation)
+ {
+ app.MapServiceControlFederation();
+ }
if (settings.EnableIntegratedServicePulse)
{
app.UseServicePulse(settings.ServicePulseSettings);
diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs
index fad7491699..54d50d8463 100644
--- a/src/ServiceControl/Infrastructure/Settings/Settings.cs
+++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs
@@ -11,6 +11,7 @@
using NServiceBus.Transport;
using Particular.ServiceControl;
using ServiceControl.Configuration;
+ using ServiceControl.Federation.Configuration;
using ServiceControl.Infrastructure;
using ServiceControl.Infrastructure.Ingestion;
using ServiceControl.Infrastructure.Settings;
@@ -68,6 +69,13 @@ public Settings(
MaximumConcurrencyLevel = SettingsReader.Read(SettingsRootNamespace, MaximumConcurrencyLevelKey);
RetryHistoryDepth = SettingsReader.Read(SettingsRootNamespace, RetryHistoryDepthKey, 10);
AllowMessageEditing = SettingsReader.Read(SettingsRootNamespace, "AllowMessageEditing");
+ EnableFederation = SettingsReader.Read(SettingsRootNamespace, "EnableFederation", false);
+ FederatedInstances = FederationSettings.ParseMembers(SettingsReader.Read(SettingsRootNamespace, "FederatedInstances"));
+ if (EnableFederation && ValidateConfiguration)
+ {
+ CreateFederationSettings().Validate();
+ }
+
EnableIntegratedServicePulse = SettingsReader.Read(SettingsRootNamespace, "EnableIntegratedServicePulse", false);
if (EnableIntegratedServicePulse)
{
@@ -76,6 +84,17 @@ public Settings(
ServiceControlUrl = "/api/",
IsIntegrated = true
};
+
+ // The embedded ServicePulse shows the federated view; /api/ remains this instance's own API,
+ // which the federation itself calls when this instance is one of the members.
+ if (EnableFederation)
+ {
+ ServicePulseSettings = ServicePulseSettings with
+ {
+ ServiceControlUrl = $"{FederationSettings.DefaultPrimaryPathBase}/",
+ MonitoringUrl = CreateFederationSettings().HasMonitoring ? $"{FederationSettings.DefaultMonitoringPathBase}/" : ServicePulseSettings.MonitoringUrl
+ };
+ }
}
NotificationsFilter = SettingsReader.Read(SettingsRootNamespace, "NotificationsFilter");
RemoteInstances = GetRemoteInstances().ToArray();
@@ -122,6 +141,19 @@ public Settings(
public bool EnableIntegratedServicePulse { get; set; }
public ServicePulseSettings ServicePulseSettings { get; set; }
+ ///
+ /// Hosts the federated API, which presents several ServiceControl installations as one, under
+ /// .
+ ///
+ public bool EnableFederation { get; set; }
+
+ ///
+ /// The installations to federate, as JSON in the style of RemoteInstances. The first is the lead member.
+ ///
+ public MemberSettings[] FederatedInstances { get; set; }
+
+ public FederationSettings CreateFederationSettings() => new() { Members = [.. FederatedInstances] };
+
//HINT: acceptance tests only
public Func MessageFilter { get; set; }
diff --git a/src/ServiceControl/ServiceControl.csproj b/src/ServiceControl/ServiceControl.csproj
index 7afd8f88cf..e9b9942b93 100644
--- a/src/ServiceControl/ServiceControl.csproj
+++ b/src/ServiceControl/ServiceControl.csproj
@@ -19,6 +19,7 @@
+