[DO NOT MERGE] Spike having a federation layer on top of multiple instances of ServiceControl

src/ServiceControl.Federation.UnitTests/.editorconfig

@@ -0,0 +1,7 @@
+[*.cs]
+
+# Justification: Application synchronization contexts don't require ConfigureAwait(false)
+dotnet_diagnostic.CA2007.severity = none
+
+# Justification: test helpers that start in-process hosts do not need cancellation
+dotnet_diagnostic.PS0018.severity = none

src/ServiceControl.Federation.UnitTests/FederationSettingsTests.cs

@@ -0,0 +1,58 @@
+namespace ServiceControl.Federation.UnitTests;
+
+using System;
+using System.Linq;
+using Configuration;
+using NUnit.Framework;
+
+[TestFixture]
+class FederationSettingsTests
+{
+    [Test]
+    public void Instances_are_parsed_in_the_remote_instances_style()
+    {
+        var members = FederationSettings.ParseMembers("""
+            [
+              { "api_uri": "http://sc-eu:33333/api/", "monitoring_uri": "http://mon-eu:33633/", "key": "eu", "name": "Europe" },
+              { "api_uri": "http://sc-us:33333/api/" }
+            ]
+            """);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(members.Select(m => m.Key), Is.EqualTo(new[] { "eu", "sc-us-33333" }), "a missing key is derived from the api_uri host and port");
+            Assert.That(members[0].Name, Is.EqualTo("Europe"));
+            Assert.That(members[0].MonitoringUrl, Is.EqualTo("http://mon-eu:33633/"));
+            Assert.That(members[1].MonitoringUrl, Is.Null);
+        }
+    }
+
+    [TestCase(null)]
+    [TestCase("")]
+    [TestCase("[]")]
+    public void No_instances_fail_validation(string? json)
+    {
+        var settings = new FederationSettings { Members = [.. FederationSettings.ParseMembers(json)] };
+
+        Assert.That(settings.Validate, Throws.InvalidOperationException.With.Message.Contains("FederatedInstances"));
+    }
+
+    [Test]
+    public void Instances_that_derive_the_same_key_must_be_given_distinct_keys()
+    {
+        var settings = new FederationSettings
+        {
+            Members = [.. FederationSettings.ParseMembers("""[{ "api_uri": "http://localhost:33333/api/" }, { "api_uri": "http://localhost:33333/api" }]""")]
+        };
+
+        Assert.That(settings.Validate, Throws.InvalidOperationException.With.Message.Contains("distinct"));
+    }
+
+    [Test]
+    public void Invalid_urls_fail_validation()
+    {
+        var settings = new FederationSettings { Members = [new MemberSettings { Key = "a", PrimaryUrl = "not a url" }] };
+
+        Assert.That(settings.Validate, Throws.InstanceOf<InvalidOperationException>().With.Message.Contains("api_uri"));
+    }
+}

src/ServiceControl.Federation.UnitTests/MergerTests.cs

@@ -0,0 +1,163 @@
+namespace ServiceControl.Federation.UnitTests;
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json.Nodes;
+using Aggregation;
+using NUnit.Framework;
+using Routing;
+
+[TestFixture]
+class FailureGroupMergerTests
+{
+    [Test]
+    public void Groups_with_the_same_id_are_combined()
+    {
+        var merged = FailureGroupMerger.MergeLists(
+        [
+            [Group("g1", 3, "2024-01-01T00:00:00Z", "2024-01-05T00:00:00Z")],
+            [Group("g1", 4, "2023-12-01T00:00:00Z", "2024-01-02T00:00:00Z"), Group("g2", 1, "2024-02-01T00:00:00Z", "2024-02-01T00:00:00Z")]
+        ]);
+
+        Assert.That(merged, Has.Count.EqualTo(2));
+        var g1 = merged.Single(g => JsonValues.Text(g, "id") == "g1");
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(JsonValues.Number(g1, "count"), Is.EqualTo(7));
+            Assert.That(JsonValues.Text(g1, "first"), Is.EqualTo("2023-12-01T00:00:00Z"));
+            Assert.That(JsonValues.Text(g1, "last"), Is.EqualTo("2024-01-05T00:00:00Z"));
+            Assert.That(JsonValues.Text(merged[0], "id"), Is.EqualTo("g2"), "ordered by last, newest first");
+        }
+    }
+
+    [Test]
+    public void Operation_state_reflects_the_most_active_member()
+    {
+        var done = Group("g1", 0, null, null);
+        done["operation_status"] = "Completed";
+        done["operation_progress"] = 1.0;
+        done["operation_remaining_count"] = 0;
+        done["operation_messages_completed_count"] = 30;
+        done["operation_completion_time"] = "2024-01-01T00:00:00Z";
+
+        var running = Group("g1", 10, null, null);
+        running["operation_status"] = "Forwarding";
+        running["operation_progress"] = 0.0;
+        running["operation_remaining_count"] = 10;
+        running["operation_messages_completed_count"] = 0;
+
+        var idle = Group("g1", 5, null, null);
+        idle["operation_status"] = "None";
+
+        var merged = FailureGroupMerger.Merge([done, running, idle]);
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(JsonValues.Text(merged, "operation_status"), Is.EqualTo("Forwarding"));
+            Assert.That(JsonValues.Number(merged, "operation_progress"), Is.EqualTo(0.75));
+            Assert.That(JsonValues.Number(merged, "operation_remaining_count"), Is.EqualTo(10));
+            Assert.That(merged["operation_completion_time"], Is.Null, "not complete until every member is");
+        }
+    }
+
+    static JsonObject Group(string id, int count, string? first, string? last) => new()
+    {
+        ["id"] = id,
+        ["title"] = "System.Exception",
+        ["type"] = "Exception Type and Stack Trace",
+        ["count"] = count,
+        ["first"] = first,
+        ["last"] = last,
+        ["operation_status"] = "None"
+    };
+}
+
+[TestFixture]
+class MonitoredEndpointMergerTests
+{
+    [Test]
+    public void Endpoints_with_the_same_name_are_combined()
+    {
+        var merged = MonitoredEndpointMerger.MergeList(
+        [
+            [Endpoint("Sales", stale: false, throughput: [10, 10], processingTime: [100, 100])],
+            [Endpoint("Sales", stale: true, throughput: [30, 0], processingTime: [200, 0]), Endpoint("Billing", stale: true, throughput: [1, 1], processingTime: [1, 1])]
+        ]);
+
+        Assert.That(merged, Has.Count.EqualTo(2));
+        var sales = merged[0]!;
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(JsonValues.Bool(sales, "isStale"), Is.False);
+            Assert.That(Points(sales, "throughput"), Is.EqualTo(new[] { 40.0, 10.0 }));
+            Assert.That(Points(sales, "processingTime"), Is.EqualTo(new[] { 175.0, 100.0 }), "weighted by throughput");
+            Assert.That((JsonValues.At(sales, "endpointInstanceIds") as JsonArray)!, Has.Count.EqualTo(2));
+        }
+    }
+
+    static IEnumerable<double?> Points(JsonNode endpoint, string metric) =>
+        (JsonValues.At(endpoint, $"metrics.{metric}.points") as JsonArray)!.Select(JsonValues.AsNumber);
+
+    static JsonObject Endpoint(string name, bool stale, double[] throughput, double[] processingTime) => new()
+    {
+        ["name"] = name,
+        ["isStale"] = stale,
+        ["endpointInstanceIds"] = new JsonArray($"{name}-{stale}"),
+        ["metrics"] = new JsonObject
+        {
+            ["throughput"] = new JsonObject { ["average"] = throughput.Average(), ["points"] = new JsonArray([.. throughput.Select(p => (JsonNode)p)]) },
+            ["processingTime"] = new JsonObject { ["average"] = processingTime.Average(), ["points"] = new JsonArray([.. processingTime.Select(p => (JsonNode)p)]) }
+        }
+    };
+}
+
+[TestFixture]
+class InstanceIdCodecTests
+{
+    [TestCase("/messages/1/body?instance_id=aHR0cDovL2E.", "a", "/messages/1/body?instance_id=a~aHR0cDovL2E.")]
+    [TestCase("/messages/1/body", "a", "/messages/1/body?instance_id=a~")]
+    [TestCase("/messages/1/body?x=1", "b", "/messages/1/body?x=1&instance_id=b~")]
+    public void Body_urls_carry_the_member_key(string url, string member, string expected) =>
+        Assert.That(InstanceIdCodec.RewriteBodyUrl(url, member), Is.EqualTo(expected));
+
+    [Test]
+    public void Encoded_ids_round_trip()
+    {
+        Assert.That(InstanceIdCodec.TryDecode(InstanceIdCodec.Encode("eu", "aHR0cDovL2E."), out var key, out var id), Is.True);
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(key, Is.EqualTo("eu"));
+            Assert.That(id, Is.EqualTo("aHR0cDovL2E."));
+        }
+
+        Assert.That(InstanceIdCodec.TryDecode("aHR0cDovL2E.", out _, out _), Is.False, "a member's own id is not routable");
+    }
+
+    [Test]
+    public void Trees_are_rewritten_in_place()
+    {
+        var row = new JsonObject
+        {
+            ["instance_id"] = "abc",
+            ["body_url"] = "/messages/1/body?instance_id=abc",
+            ["nested"] = new JsonArray(new JsonObject { ["body_url"] = "/messages/2/body" })
+        };
+
+        InstanceIdCodec.RewriteTree(row, "eu");
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(JsonValues.Text(row, "instance_id"), Is.EqualTo("eu~abc"));
+            Assert.That(JsonValues.Text(row, "body_url"), Is.EqualTo("/messages/1/body?instance_id=eu~abc"));
+            Assert.That(JsonValues.Text(row["nested"]![0], "body_url"), Is.EqualTo("/messages/2/body?instance_id=eu~"));
+        }
+    }
+}
+
+[TestFixture]
+class VersionsTests
+{
+    [Test]
+    public void The_lowest_version_wins() =>
+        Assert.That(Versions.Lowest(["6.10.0", "6.9.1", null, "6.9.1-alpha.3+sha"]), Is.EqualTo("6.9.1"));
+}

src/ServiceControl.Federation.UnitTests/NUnitSettings.cs

@@ -0,0 +1,5 @@
+using NUnit.Framework;
+
+[assembly: FixtureLifeCycle(LifeCycle.InstancePerTestCase)]
+[assembly: Parallelizable(ParallelScope.All)]
+[assembly: LevelOfParallelism(4)]

src/ServiceControl.Federation.UnitTests/RouteCoverageTests.cs

@@ -0,0 +1,91 @@
+namespace ServiceControl.Federation.UnitTests;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Api;
+using Members;
+using NUnit.Framework;
+
+/// <summary>
+/// Every route a ServiceControl instance exposes must have a federation strategy. The approved route
+/// lists are the instances' own approval files, so adding a route to ServiceControl fails this test
+/// until the route is classified in <see cref="PrimaryRoutes"/> or <see cref="MonitoringRoutes"/>.
+/// </summary>
+[TestFixture]
+class RouteCoverageTests
+{
+    [Test]
+    public void Every_primary_route_has_a_strategy()
+    {
+        var approved = ApprovedRoutes("Primary.txt", PrimaryPrefix)
+            .Concat(ApprovedRoutes("Licensing.txt", _ => "licensing/"));
+
+        AssertCovered(approved, PrimaryRoutes.All);
+    }
+
+    [Test]
+    public void Every_monitoring_route_has_a_strategy() =>
+        AssertCovered(ApprovedRoutes("Monitoring.txt", MonitoringPrefix), MonitoringRoutes.All);
+
+    [Test]
+    public void Route_tables_have_no_duplicates()
+    {
+        var duplicates = FederationHostingExtensions.AllRoutes
+            .SelectMany(route => route.Methods.Select(method => $"{route.Surface} {method} {Normalise(route.Template)}"))
+            .GroupBy(key => key)
+            .Where(group => group.Count() > 1)
+            .Select(group => group.Key)
+            .ToList();
+
+        Assert.That(duplicates, Is.Empty);
+    }
+
+    static void AssertCovered(IEnumerable<string> approved, IEnumerable<FederatedRoute> routes)
+    {
+        var proxied = routes
+            .SelectMany(route => route.Methods.Select(method => $"{method} {Normalise(route.Template)}"))
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+
+        var missing = approved.Where(route => !proxied.Contains(route)).ToList();
+
+        Assert.That(missing, Is.Empty, "These routes have no federation strategy. Add them to the route table.");
+    }
+
+    // The approval files print only the method-level template; these controllers add a prefix.
+    static string PrimaryPrefix(string controller) => controller switch
+    {
+        _ when controller.EndsWith("AuthenticationController", StringComparison.Ordinal) => "authentication/",
+        _ when controller.EndsWith("SystemMaintenanceController", StringComparison.Ordinal) => "maintenance/",
+        _ => ""
+    };
+
+    static string MonitoringPrefix(string controller) =>
+        controller.EndsWith("MyRoutesController", StringComparison.Ordinal) ? "api/" : "";
+
+    static IEnumerable<string> ApprovedRoutes(string file, Func<string, string> prefixFor)
+    {
+        var path = Path.Combine(TestContext.CurrentContext.TestDirectory, "ApprovedRoutes", file);
+        foreach (var line in File.ReadAllLines(path).Where(l => l.Contains("=>", StringComparison.Ordinal)))
+        {
+            var match = RouteLine().Match(line);
+            Assert.That(match.Success, Is.True, $"Unexpected approval line: {line}");
+
+            var controller = match.Groups["controller"].Value;
+            var template = prefixFor(controller) + match.Groups["template"].Value.TrimStart('/');
+            foreach (var method in match.Groups["methods"].Value.Split('/'))
+            {
+                yield return $"{method} {Normalise(template)}";
+            }
+        }
+    }
+
+    static string Normalise(string template) =>
+        string.Join('/', template.Trim('/').Split('/').Select(segment => Parameter().Replace(segment, "{}"))).ToLowerInvariant();
+
+    static Regex RouteLine() => new(@"^(?<methods>[A-Z/]+) (?<template>\S+) => (?<controller>[^:]+):");
+
+    static Regex Parameter() => new(@"\{[^}]*\}");
+}

src/ServiceControl.Federation.UnitTests/ServiceControl.Federation.UnitTests.csproj

@@ -0,0 +1,33 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <TestCategory>DefaultCore</TestCategory>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ServiceControl.Federation\ServiceControl.Federation.csproj" />
+    <ProjectReference Include="..\TestHelper\TestHelper.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="GitHubActionsTestLogger" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="NUnit" />
+    <PackageReference Include="NUnit.Analyzers" />
+    <PackageReference Include="NUnit3TestAdapter" />
+  </ItemGroup>
+
+  <!-- The route coverage test compares the proxy's route tables with the instances' approved route lists -->
+  <ItemGroup>
+    <None Include="..\ServiceControl.UnitTests\ApprovalFiles\APIApprovals.HttpApiRoutes.approved.txt" Link="ApprovedRoutes\Primary.txt" CopyToOutputDirectory="PreserveNewest" />
+    <None Include="..\Particular.LicensingComponent.UnitTests\ApprovalFiles\APIApprovals.HttpApiRoutes.approved.txt" Link="ApprovedRoutes\Licensing.txt" CopyToOutputDirectory="PreserveNewest" />
+    <None Include="..\ServiceControl.Monitoring.UnitTests\ApprovalFiles\APIApprovals.HttpApiRoutes.approved.txt" Link="ApprovedRoutes\Monitoring.txt" CopyToOutputDirectory="PreserveNewest" />
+  </ItemGroup>
+
+</Project>