Add optional PerfLab counter regression thresholds

src/tools/Reporting/Directory.Packages.props

@@ -4,7 +4,7 @@
     <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
   </PropertyGroup>
   <ItemGroup>
-    <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageVersion Include="System.Text.Json" Version="10.0.11" />
     <PackageVersion Include="Microsoft.DotNet.PlatformAbstractions" Version="2.1.0" />
   </ItemGroup>
 </Project>

src/tools/Reporting/Reporting.Tests/ReporterTests.cs

@@ -2,9 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Newtonsoft.Json;
 using System;
 using System.Runtime.InteropServices;
+using System.Text.Json;
 using Xunit;
 
 namespace Reporting.Tests;
@@ -30,7 +30,6 @@ public class ReporterTests
 ---------------|-------------------------|-------------------------|-------------------------
 CounterName    |10000000000000000.000 ns |10000000000000000.000 ns |10000000000000000.000 ns 
 ";
-
     private const string NoResultsTable =
 @"TestName
 No results in file.
@@ -57,6 +56,7 @@ public void WriteReportTableWithEmptyResults()
                 new Counter
                 {
                     DefaultCounter = true,
+                    TopCounter = true,
                     MetricName = "ns",
                     Name = "CounterName",
                     Results = []
@@ -79,6 +79,7 @@ public void WriteReportTableWithNullResults()
                 new Counter
                 {
                     DefaultCounter = true,
+                    TopCounter = true,
                     MetricName = "ns",
                     Name = "CounterName",
                     Results = null
@@ -119,7 +120,7 @@ public void WriteReportTableWithoutEnvironment()
     public void WriteReportWithLongNameTableWithoutEnvironment()
     {
         PerfLabEnvironmentProviderMock environment = new NonPerfLabEnvironmentProviderMock();
-        var reporter = GetReporterWithSpecifiedEnvironment(environment, counterName:"ThisIsALongerCounterName");
+        var reporter = GetReporterWithSpecifiedEnvironment(environment, counterName: "ThisIsALongerCounterName");
         var table = reporter.WriteResultTable();
         Assert.Equal(LongCounterNameTable, table);
     }
@@ -139,8 +140,9 @@ public void JsonCanBeGenerated()
         var environment = new PerfLabEnvironmentProviderMock();
         var reporter = GetReporterWithSpecifiedEnvironment(environment);
         var jsonString = reporter.GetJson();
+        Assert.NotNull(jsonString);
 
-        var jsonObj = JsonConvert.DeserializeObject<Reporter>(jsonString);
+        var jsonObj = DeserializeReporter(jsonString);
 
         Assert.Equal(environment.GetEnvironmentVariable("HELIX_CORRELATION_ID"), jsonObj.Run.CorrelationId);
         Assert.Equal(environment.GetEnvironmentVariable("HELIX_WORKITEM_FRIENDLYNAME"), jsonObj.Run.WorkItemName);
@@ -168,6 +170,55 @@ public void JsonCanBeGenerated()
         Assert.Equal(1.1, retCounter.Results[0]);
     }
 
+    [Fact]
+    public void LegacyCounterJsonRemainsCompatible()
+    {
+        var reporter = GetReporterWithSpecifiedEnvironment(new PerfLabEnvironmentProviderMock());
+        var jsonString = reporter.GetJson();
+        Assert.NotNull(jsonString);
+
+        using var document = JsonDocument.Parse(jsonString);
+        var jsonCounter = document.RootElement.GetProperty("tests")[0].GetProperty("counters")[0];
+
+        Assert.Equal(JsonValueKind.False, jsonCounter.GetProperty("higherIsBetter").ValueKind);
+        Assert.False(jsonCounter.TryGetProperty("regressionThreshold", out _));
+        Assert.False(jsonCounter.TryGetProperty("direction", out _));
+
+        var deserialized = DeserializeReporter(jsonString);
+        Assert.False(deserialized.Tests[0].Counters[0].HigherIsBetter);
+        Assert.Null(deserialized.Tests[0].Counters[0].RegressionThreshold);
+    }
+
+    [Fact]
+    public void RegressionThresholdIsSerialized()
+    {
+        var reporter = GetReporterWithSpecifiedEnvironment(new PerfLabEnvironmentProviderMock());
+        reporter.Tests[0].Counters[0].RegressionThreshold = 0.02;
+
+        var jsonString = reporter.GetJson();
+        Assert.NotNull(jsonString);
+
+        using var document = JsonDocument.Parse(jsonString);
+        var jsonCounter = document.RootElement.GetProperty("tests")[0].GetProperty("counters")[0];
+        Assert.Equal(0.02, jsonCounter.GetProperty("regressionThreshold").GetDouble());
+
+        var deserialized = DeserializeReporter(jsonString);
+        Assert.Equal(0.02, deserialized.Tests[0].Counters[0].RegressionThreshold);
+    }
+
+    [Fact]
+    public void NonFiniteResultsRoundTrip()
+    {
+        var reporter = GetReporterWithSpecifiedEnvironment(new PerfLabEnvironmentProviderMock(), result: double.NaN);
+
+        var jsonString = reporter.GetJson();
+        Assert.NotNull(jsonString);
+        Assert.Contains("\"NaN\"", jsonString);
+
+        var deserialized = DeserializeReporter(jsonString);
+        Assert.True(double.IsNaN(deserialized.Tests[0].Counters[0].Results[0]));
+    }
+
     [Fact]
     public void EnforceDefaultCounterConstraint()
     {
@@ -205,6 +256,9 @@ public void AddCountersEnumerable()
         Assert.Equal(2, t.Counters.Count);
     }
 
+    private static Reporter DeserializeReporter(string json)
+        => Reporter.FromJson(json);
+
     private static Reporter GetReporterWithSpecifiedEnvironment(PerfLabEnvironmentProviderMock enviroment, string counterName = null, double result = 1.1)
     {
         var reporter = new Reporter(enviroment);

src/tools/Reporting/Reporting/Counter.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System.Collections.Generic;
+using System.Text.Json.Serialization;
 
 namespace Reporting;
 
@@ -18,6 +19,9 @@ public class Counter
 
     public string MetricName { get; set; } = "Count";
 
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public double? RegressionThreshold { get; set; }
+
     public IList<double>? Results { get; set; }
 
     public override string ToString() => $"{nameof(Name)}: {Name}, {nameof(TopCounter)}: {TopCounter}, {nameof(DefaultCounter)}: {DefaultCounter}, {nameof(MetricName)}: {MetricName}";

src/tools/Reporting/Reporting/Reporter.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Newtonsoft.Json;
-using Newtonsoft.Json.Serialization;
 using System;
 using System.Collections;
 using System.Collections.Generic;
@@ -12,20 +10,20 @@
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using RuntimeEnvironment = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment;
 
 namespace Reporting;
 
 public class Reporter
 {
     private readonly static CultureInfo _culture = CultureInfo.InvariantCulture;
-    private readonly static JsonSerializerSettings _jsonSerializerSettings = new()
+    private readonly static JsonSerializerOptions _jsonSerializerOptions = new()
     {
-        ContractResolver = new DefaultContractResolver
-        {
-            NamingStrategy = new CamelCaseNamingStrategy() { ProcessDictionaryKeys = false }
-        },
-        Culture = _culture
+        NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals,
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        WriteIndented = true
     };
 
     public List<Test> Tests { get; private set; } = [];
@@ -65,7 +63,18 @@ public void AddTest(Test test)
     }
 
     public string? GetJson()
-        => InLab ? JsonConvert.SerializeObject(this, Formatting.Indented, _jsonSerializerSettings) : null;
+    {
+        if (!InLab)
+        {
+            return null;
+        }
+
+        return JsonSerializer.Serialize(this, _jsonSerializerOptions);
+    }
+
+    public static Reporter FromJson(string json)
+        => JsonSerializer.Deserialize<Reporter>(json, _jsonSerializerOptions)
+            ?? throw new JsonException("The JSON payload did not contain a reporter.");
 
     public string WriteResultTable()
     {
@@ -81,7 +90,7 @@ public string WriteResultTable()
 
             var countersWithResults = test.Counters.Where(c => c.Results != null && c.Results.Count > 0);
             var counterWidth = Math.Max(test.Counters.Max(c => c.Name.Length) + 1, 15);
-            var resultWidth = Math.Max(countersWithResults.Max(c => c.Results.Max().ToString("F3", _culture).Length + c.MetricName.Length) + 2, 15);
+            var resultWidth = Math.Max(countersWithResults.Max(c => c.Results.Max().ToString("F3", _culture).Length + (c.MetricName?.Length ?? 0)) + 2, 15);
             ret.AppendLine(test.Name);
             ret.AppendLine($"{LeftJustify("Metric", counterWidth)}|{LeftJustify("Average", resultWidth)}|{LeftJustify("Min", resultWidth)}|{LeftJustify("Max", resultWidth)}");
             ret.AppendLine($"{new string('-', counterWidth)}|{new string('-', resultWidth)}|{new string('-', resultWidth)}|{new string('-', resultWidth)}");

src/tools/Reporting/Reporting/Reporting.csproj

@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Newtonsoft.Json" />
+    <PackageReference Include="System.Text.Json" />
     <PackageReference Include="Microsoft.DotNet.PlatformAbstractions" />
   </ItemGroup>