diff --git a/.github/workflows/tools-tests.yaml b/.github/workflows/tools-tests.yaml index 3bac9dfca..aa0a65bc6 100644 --- a/.github/workflows/tools-tests.yaml +++ b/.github/workflows/tools-tests.yaml @@ -33,4 +33,4 @@ jobs: - name: Run all tests run: | cd tools - dotnet test + dotnet test --filter Name!~ExampleTests diff --git a/tools/ExampleExtractor/ExampleMetadata.cs b/tools/ExampleExtractor/ExampleMetadata.cs index a5b79fc56..cd52a7cbe 100644 --- a/tools/ExampleExtractor/ExampleMetadata.cs +++ b/tools/ExampleExtractor/ExampleMetadata.cs @@ -84,4 +84,6 @@ public class ExampleMetadata [JsonIgnore] public string Source => $"{MarkdownFile}:{StartLine}-{EndLine}"; + + public override string ToString() => Name; } diff --git a/tools/ExampleTester/ExampleTester.csproj b/tools/ExampleTester/ExampleTester.csproj index 8642c9cad..4d75197c2 100644 --- a/tools/ExampleTester/ExampleTester.csproj +++ b/tools/ExampleTester/ExampleTester.csproj @@ -1,15 +1,19 @@ - + Exe net8.0 enable enable + false + + + diff --git a/tools/ExampleTester/ExampleTests.cs b/tools/ExampleTester/ExampleTests.cs new file mode 100644 index 000000000..f772bc032 --- /dev/null +++ b/tools/ExampleTester/ExampleTests.cs @@ -0,0 +1,35 @@ +using NUnit.Framework; +using Utilities; + +[assembly: Parallelizable(ParallelScope.Children)] + +namespace ExampleTester; + +public static class ExampleTests +{ + private static TesterConfiguration TesterConfiguration { get; } = new(Path.Join(FindSlnDirectory(), "tmp")); + + public static IEnumerable LoadExamples() => + from example in GeneratedExample.LoadAllExamples(TesterConfiguration.ExtractedOutputDirectory) + select new object[] { example }; + + [TestCaseSource(nameof(LoadExamples))] + public static async Task ExamplePasses(GeneratedExample example) + { + var logger = new StatusCheckLogger(TestContext.Out, "..", "Example tester"); + + if (!await example.Test(TesterConfiguration, logger)) + Assert.Fail("There were one or more failures. See the logged output for details."); + } + + private static string FindSlnDirectory() + { + for (string? current = AppDomain.CurrentDomain.BaseDirectory; current != null; current = Path.GetDirectoryName(current)) + { + if (Directory.EnumerateFiles(current, "*.sln").Any()) + return current; + } + + throw new InvalidOperationException($"Can't find a directory containing a .sln file in {AppDomain.CurrentDomain.BaseDirectory} or any parent directories."); + } +} diff --git a/tools/ExampleTester/GeneratedExample.cs b/tools/ExampleTester/GeneratedExample.cs index aeea299d9..68a3b19cc 100644 --- a/tools/ExampleTester/GeneratedExample.cs +++ b/tools/ExampleTester/GeneratedExample.cs @@ -8,8 +8,10 @@ namespace ExampleTester; -internal class GeneratedExample +public class GeneratedExample { + private static readonly object ConsoleAccessLock = new(); + private readonly string directory; internal ExampleMetadata Metadata { get; } @@ -20,6 +22,8 @@ private GeneratedExample(string directory) Metadata = JsonConvert.DeserializeObject(metadataJson) ?? throw new ArgumentException($"Invalid (null) metadata in {directory}"); } + public override string? ToString() => Metadata.ToString(); + internal static List LoadAllExamples(string parentDirectory) => Directory.GetDirectories(parentDirectory).Select(Load).ToList(); @@ -127,46 +131,52 @@ bool ValidateOutput() ? new object[] { Metadata.ExecutionArgs ?? new string[0] } : new object[0]; - var oldOut = Console.Out; List actualLines; Exception? actualException = null; - try + lock (ConsoleAccessLock) { - var builder = new StringBuilder(); - Console.SetOut(new StringWriter(builder)); + var oldOut = Console.Out; try { - var result = method.Invoke(null, arguments); - // For async Main methods, the compilation's entry point is still the Main - // method, so we explicitly wait for the returned task just like the synthesized - // entry point would. - if (result is Task task) + var builder = new StringBuilder(); + Console.SetOut(new StringWriter(builder)); + try + { + var result = method.Invoke(null, arguments); + // For async Main methods, the compilation's entry point is still the Main + // method, so we explicitly wait for the returned task just like the synthesized + // entry point would. + if (result is Task task) + { + task.GetAwaiter().GetResult(); + } + + // For some reason, we don't *actually* get the result of all finalizers + // without this. We shouldn't need it (as relevant examples already have it) but + // code that works outside the test harness doesn't work inside it. + GC.Collect(); + GC.WaitForPendingFinalizers(); + } + catch (TargetInvocationException outer) { - task.GetAwaiter().GetResult(); + actualException = outer.InnerException ?? throw new InvalidOperationException("TargetInvocationException had no nested exception"); } - // For some reason, we don't *actually* get the result of all finalizers - // without this. We shouldn't need it (as relevant examples already have it) but - // code that works outside the test harness doesn't work inside it. - GC.Collect(); - GC.WaitForPendingFinalizers(); + + // Skip blank lines, to avoid unnecessary trailing empties. + // Also trim the end of each actual line, to avoid trailing spaces being necessary in the metadata + // or listed console output. + actualLines = builder.ToString() + .Replace("\r\n", "\n") + .Split('\n') + .Select(line => line.TrimEnd()) + .Where(line => line != "").ToList(); } - catch (TargetInvocationException outer) + finally { - actualException = outer.InnerException ?? throw new InvalidOperationException("TargetInvocationException had no nested exception"); + Console.SetOut(oldOut); } - // Skip blank lines, to avoid unnecessary trailing empties. - // Also trim the end of each actual line, to avoid trailing spaces being necessary in the metadata - // or listed console output. - actualLines = builder.ToString() - .Replace("\r\n", "\n") - .Split('\n') - .Select(line => line.TrimEnd()) - .Where(line => line != "").ToList(); - } - finally - { - Console.SetOut(oldOut); } + var expectedLines = Metadata.ExpectedOutput ?? new List(); return ValidateException(actualException, Metadata.ExpectedException) && (Metadata.IgnoreOutput || ValidateExpectedAgainstActual("output", expectedLines, actualLines)); diff --git a/tools/ExampleTester/Program.cs b/tools/ExampleTester/Program.cs index 937a79e3d..7486d036a 100644 --- a/tools/ExampleTester/Program.cs +++ b/tools/ExampleTester/Program.cs @@ -2,7 +2,7 @@ using System.CommandLine; using Utilities; -var logger = new StatusCheckLogger("..", "Example tester"); +var logger = new StatusCheckLogger(Console.Out, "..", "Example tester"); var headSha = Environment.GetEnvironmentVariable("HEAD_SHA"); var token = Environment.GetEnvironmentVariable("GH_TOKEN"); diff --git a/tools/ExampleTester/TesterConfiguration.cs b/tools/ExampleTester/TesterConfiguration.cs index b6a0c43d9..eee1dff14 100644 --- a/tools/ExampleTester/TesterConfiguration.cs +++ b/tools/ExampleTester/TesterConfiguration.cs @@ -5,12 +5,9 @@ namespace ExampleTester; public record TesterConfiguration( string ExtractedOutputDirectory, - bool Quiet, - string? SourceFile, - string? ExampleName) -{ - -} + bool Quiet = false, + string? SourceFile = null, + string? ExampleName = null); public class TesterConfigurationBinder : BinderBase { diff --git a/tools/MarkdownConverter/Spec/Reporter.cs b/tools/MarkdownConverter/Spec/Reporter.cs index 4adaa52c8..58930aa45 100644 --- a/tools/MarkdownConverter/Spec/Reporter.cs +++ b/tools/MarkdownConverter/Spec/Reporter.cs @@ -28,7 +28,7 @@ public Reporter() : this(null, null) { } public Reporter(Reporter? parent, string? filename) { // This is needed so that all Reporters share the same GitHub logger. - this.githubLogger = parent?.githubLogger ?? new StatusCheckLogger("..", "Markdown to Word Converter"); + this.githubLogger = parent?.githubLogger ?? new StatusCheckLogger(Console.Out, "..", "Markdown to Word Converter"); this.parent = parent; Location = new SourceLocation(filename, null, null, null); } diff --git a/tools/StandardAnchorTags/Program.cs b/tools/StandardAnchorTags/Program.cs index 6c0f77449..fea1ffd7b 100644 --- a/tools/StandardAnchorTags/Program.cs +++ b/tools/StandardAnchorTags/Program.cs @@ -25,7 +25,7 @@ public class Program /// 0 on success, non-zero on failure static async Task Main(string owner, string repo, bool dryrun =false) { - var logger = new StatusCheckLogger("..", "TOC and Anchor updater"); + var logger = new StatusCheckLogger(Console.Out, "..", "TOC and Anchor updater"); var headSha = Environment.GetEnvironmentVariable("HEAD_SHA"); var token = Environment.GetEnvironmentVariable("GH_TOKEN"); using FileStream openStream = File.OpenRead(FilesPath); diff --git a/tools/Utilities/StatusCheckLogger.cs b/tools/Utilities/StatusCheckLogger.cs index 2feaeb65c..dc8e83c4f 100644 --- a/tools/Utilities/StatusCheckLogger.cs +++ b/tools/Utilities/StatusCheckLogger.cs @@ -22,7 +22,7 @@ public record StatusCheckMessage(string file, int StartLine, int EndLine, string /// /// The path to the root of the repository /// The name of the tool that is running the check -public class StatusCheckLogger(string pathToRoot, string toolName) +public class StatusCheckLogger(TextWriter writer, string pathToRoot, string toolName) { private List annotations = []; public bool Success { get; private set; } = true; @@ -30,7 +30,7 @@ public class StatusCheckLogger(string pathToRoot, string toolName) // Utility method to format the path to unix style, from the root of the repository. private string FormatPath(string path) => Path.GetRelativePath(pathToRoot, path).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); - private void WriteMessageToConsole(string prefix, StatusCheckMessage d) => Console.WriteLine($"{prefix}{toolName}-{d.Id}::file={FormatPath(d.file)},line={d.StartLine}::{d.Message}"); + private void WriteMessageToConsole(string prefix, StatusCheckMessage d) => writer.WriteLine($"{prefix}{toolName}-{d.Id}::file={FormatPath(d.file)},line={d.StartLine}::{d.Message}"); /// /// Log a notice from the status check to the console only @@ -178,9 +178,9 @@ public async Task BuildCheckRunResult(string token, string owner, string repo, s // Once running on a branch on the dotnet org, this should work correctly. catch (ForbiddenException e) { - Console.WriteLine("===== WARNING: Could not create a check run.====="); - Console.WriteLine("Exception details:"); - Console.WriteLine(e); + writer.WriteLine("===== WARNING: Could not create a check run.====="); + writer.WriteLine("Exception details:"); + writer.WriteLine(e); } } }