diff --git a/src/tools/auto-bisect/Directory.Build.props b/src/tools/auto-bisect/Directory.Build.props new file mode 100644 index 00000000000000..8292cf3a0c8510 --- /dev/null +++ b/src/tools/auto-bisect/Directory.Build.props @@ -0,0 +1,8 @@ + + + + + + true + + diff --git a/src/tools/auto-bisect/Directory.Build.targets b/src/tools/auto-bisect/Directory.Build.targets new file mode 100644 index 00000000000000..f1b3c8f48a92f2 --- /dev/null +++ b/src/tools/auto-bisect/Directory.Build.targets @@ -0,0 +1,4 @@ + + + + diff --git a/src/tools/auto-bisect/README.md b/src/tools/auto-bisect/README.md new file mode 100644 index 00000000000000..0ac0702cbad438 --- /dev/null +++ b/src/tools/auto-bisect/README.md @@ -0,0 +1,22 @@ +# auto-bisect + +A tool for automatically finding the first commit that introduced a test failure in Azure DevOps builds using binary search. Given a known good build and a known bad build, it will automatically queue builds (or use existing ones) to test commits in between, narrowing down to the exact commit that caused the regression. + +**Requirements:** + +1. A personal AzDO PAT with read & execute permission for the "Build" tasks, and read for "Test Management". +2. A "good" build and a "bad" build, where a test pass in the good build and fail in the bad build. +3. The name of a test to track. `auto-bisect diff` can help you find tests that newly fail in between two builds. + +**Usage:** +Inside the src directory, run + +```bash +export AZDO_PAT= +dotnet run -- bisect \ + -o -p \ + --good --bad \ + --test +``` + +The public dotnet testing org is "dnceng-public" and the project is "public". \ No newline at end of file diff --git a/src/tools/auto-bisect/auto-bisect.sln b/src/tools/auto-bisect/auto-bisect.sln new file mode 100644 index 00000000000000..9789d7316d47a4 --- /dev/null +++ b/src/tools/auto-bisect/auto-bisect.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "auto-bisect", "src\auto-bisect.csproj", "{F7379BEA-FDBD-4C00-A619-05A433BB813F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoBisect.Tests", "tests\AutoBisect.Tests.csproj", "{F77542B9-DBCC-4F17-B30C-3EBD7C91B3C3}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {F7379BEA-FDBD-4C00-A619-05A433BB813F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F7379BEA-FDBD-4C00-A619-05A433BB813F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F7379BEA-FDBD-4C00-A619-05A433BB813F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F7379BEA-FDBD-4C00-A619-05A433BB813F}.Release|Any CPU.Build.0 = Release|Any CPU + {F77542B9-DBCC-4F17-B30C-3EBD7C91B3C3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F77542B9-DBCC-4F17-B30C-3EBD7C91B3C3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F77542B9-DBCC-4F17-B30C-3EBD7C91B3C3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F77542B9-DBCC-4F17-B30C-3EBD7C91B3C3}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/src/tools/auto-bisect/docs/design.md b/src/tools/auto-bisect/docs/design.md new file mode 100644 index 00000000000000..3f5747ef29e980 --- /dev/null +++ b/src/tools/auto-bisect/docs/design.md @@ -0,0 +1,105 @@ +# Auto-Bisect Tool Design + +## Overview + +Auto-bisect is a command-line tool that automatically identifies the first commit that introduced a test failure in Azure DevOps builds. Given a known good build and a known bad build, it bisects the commit range to find the exact commit that caused the regression. + +## Architecture + +### High-Level Design + +The tool integrates three key systems: +1. **Azure DevOps API** - For accessing build information, test results, and queuing new builds +2. **Local Git repository** - For retrieving commit history and metadata +3. **Bisection algorithm** - For efficiently narrowing down the search space + +### User Interaction Flow + +#### 1. Initial Setup +The user provides: +- Azure DevOps organization and project +- A known good build ID (test passes) +- A known bad build ID (test fails) +- The exact name of the test to track + +The tool authenticates using a Personal Access Token (PAT) from the `AZDO_PAT` environment variable or `--pat` flag. + +#### 2. Bisection Process + +In **auto-queue mode** (default), the tool automatically: +- Searches for existing builds at the target commit +- Queues a new build if none exists +- Polls the build until completion +- Checks the test result and updates the search range + +In **manual mode**, the tool: +- Only uses existing builds +- Prompts the user to manually queue builds for untested commits +- Waits for user confirmation before proceeding + +#### 3. Result Presentation +Once complete, the tool displays: +- The exact commit SHA that introduced the failure +- Commit metadata (author, date, message) +- Summary of all builds tested during bisection + +### Supporting Commands + +Beyond the main `bisect` command, the tool provides utilities for exploration: + +- **`diff`** - Compares two builds to show which tests newly failed, useful for finding the right test to bisect +- **`tests`** - Lists all failed tests in a specific build +- **`build`** - Shows detailed information about a single build +- **`queued`** - Displays status of queued builds for monitoring progress + +## Key Design Decisions + +### Azure DevOps Integration +Uses Azure DevOps REST API with PAT authentication. Requires minimal permissions: read access to builds and tests, plus execute permission to queue builds in auto-queue mode. + +### Local Git Dependency +Relies on a local Git clone to retrieve commit history rather than querying Azure DevOps. This provides accurate chronological ordering and avoids API limitations, but requires the user to have the repository checked out locally. + +### Stateless and resumable +The tool doesn't maintain its own persistent state between runs, but it reuses existing build results from Azure DevOps. If a bisection is interrupted and restarted with the same good/bad builds, it will find and use previously queued builds rather than queuing duplicates. This makes the bisection process resumable without explicit state management. + +## Usage Examples + +### Typical Workflow +```bash +# Set up authentication +export AZDO_PAT= + +# First, find which tests are failing +dotnet run -- diff \ + -o dnceng-public -p public \ + --good 12345 --bad 12350 + +# Then bisect to find the culprit commit +dotnet run -- bisect \ + -o dnceng-public -p public \ + --good 12345 --bad 12350 \ + --test "MyNamespace.MyTestClass.MyFailingTest" +``` + +### Manual Mode (Use Existing Builds Only) +```bash +dotnet run -- bisect \ + -o dnceng-public -p public \ + --good 12345 --bad 12350 \ + --test "MyNamespace.MyTestClass.MyFailingTest" \ + --manual +``` + +## Limitations + +- Requires a local Git repository clone at the commit range being tested +- Needs pre-existing good and bad builds as starting points +- Test name must match exactly as reported in Azure DevOps test results +- Azure DevOps specific—does not support other CI/CD platforms + +## Trade-offs + +**Build Result Reuse**: The tool leverages Azure DevOps as implicit state storage by reusing existing builds. This means bisections are naturally resumable but also means you can't easily "reset" a bisection without manually deleting builds. + +**Auto-queue vs. Manual**: Auto-queue mode is faster and more convenient but consumes CI resources. Manual mode gives users full control over when builds run, useful for resource-constrained scenarios or when builds are expensive. diff --git a/src/tools/auto-bisect/src/AzDoClient.cs b/src/tools/auto-bisect/src/AzDoClient.cs new file mode 100644 index 00000000000000..20a9fb91caf460 --- /dev/null +++ b/src/tools/auto-bisect/src/AzDoClient.cs @@ -0,0 +1,431 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; + +namespace AutoBisect; + +/// +/// Interface for Azure DevOps API client operations. +/// +public interface IAzDoClient +{ + /// + /// Gets build information by build ID. + /// + Task GetBuildAsync(int buildId, CancellationToken cancellationToken = default); + + /// + /// Gets failed test results for a specific build. + /// + IAsyncEnumerable GetFailedTestsAsync( + int buildId, + CancellationToken cancellationToken = default + ); + + /// + /// Checks if a specific test has failed in a build (even if the build is still running). + /// + Task HasTestFailedAsync( + int buildId, + string testName, + CancellationToken cancellationToken = default + ); + + /// + /// Finds completed builds for a specific commit and pipeline definition. + /// + Task> FindBuildsAsync( + string commitSha, + int? definitionId = null, + CancellationToken cancellationToken = default + ); + + /// + /// Queues a new build for a specific commit. + /// + Task QueueBuildAsync( + int definitionId, + string commitSha, + string? sourceBranch = null, + CancellationToken cancellationToken = default + ); +} + +/// +/// Azure DevOps REST API client. +/// +public class AzDoClient : IAzDoClient, IDisposable +{ + private readonly HttpClient _httpClient; + private readonly string _organization; + private readonly string _project; + + public AzDoClient(string organization, string project, string personalAccessToken) + : this(organization, project, personalAccessToken, null) { } + + internal AzDoClient( + string organization, + string project, + string personalAccessToken, + HttpClient? httpClient + ) + { + ArgumentException.ThrowIfNullOrWhiteSpace(organization); + ArgumentException.ThrowIfNullOrWhiteSpace(project); + ArgumentException.ThrowIfNullOrWhiteSpace(personalAccessToken); + + _organization = organization; + _project = project; + + _httpClient = httpClient ?? new HttpClient(); + _httpClient.BaseAddress = new Uri($"https://dev.azure.com/{_organization}/{_project}/"); + + var credentials = Convert.ToBase64String( + System.Text.Encoding.ASCII.GetBytes($":{personalAccessToken}") + ); + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + credentials + ); + _httpClient.DefaultRequestHeaders.Accept.Add( + new MediaTypeWithQualityHeaderValue("application/json") + ); + } + + public async Task GetBuildAsync( + int buildId, + CancellationToken cancellationToken = default + ) + { + var response = await _httpClient.GetAsync( + $"_apis/build/builds/{buildId}?api-version=7.1", + cancellationToken + ); + + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) + { + return null; + } + + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadFromJsonAsync( + AzDoJsonContext.Default.Build, + cancellationToken + ); + } + + public async IAsyncEnumerable GetFailedTestsAsync( + int buildId, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default + ) + { + var continuationToken = (string?)null; + + do + { + var url = $"_apis/test/runs?buildUri=vstfs:///Build/Build/{buildId}&api-version=7.1"; + url += continuationToken != null ? $"&continuationToken={continuationToken}" : ""; + + var response = await _httpClient.GetAsync(url, cancellationToken); + response.EnsureSuccessStatusCode(); + + var runsResponse = await response.Content.ReadFromJsonAsync( + AzDoJsonContext.Default.TestRunsResponse, + cancellationToken + ); + if (runsResponse?.Value is not { } runs) + { + break; + } + + foreach (var run in runs) + { + await foreach (var testResult in GetFailedTestsForRunAsync(run.Id, cancellationToken)) + { + yield return testResult; + } + } + + continuationToken = response.Headers.TryGetValues( + "x-ms-continuationtoken", + out var tokens + ) + ? tokens.FirstOrDefault() + : null; + } while (continuationToken != null); + } + + public async Task HasTestFailedAsync( + int buildId, + string testName, + CancellationToken cancellationToken = default + ) + { + try + { + var url = $"_apis/test/runs?buildUri=vstfs:///Build/Build/{buildId}&api-version=7.1"; + var response = await _httpClient.GetAsync(url, cancellationToken); + + if (!response.IsSuccessStatusCode) + { + return false; + } + + var runsResponse = await response.Content.ReadFromJsonAsync( + AzDoJsonContext.Default.TestRunsResponse, + cancellationToken + ); + + if (runsResponse?.Value is not { } runs) + { + return false; + } + + // Check each test run for the specific failure + foreach (var run in runs) + { + await foreach (var testResult in GetFailedTestsForRunAsync(run.Id, cancellationToken)) + { + if (testResult.FullyQualifiedName.Equals(testName, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + } + + return false; + } + catch + { + // If we can't check, assume not failed yet + return false; + } + } + + private async IAsyncEnumerable GetFailedTestsForRunAsync( + int runId, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken + ) + { + int skip = 0; + const int top = 1000; + + while (true) + { + var response = await _httpClient.GetAsync( + $"_apis/test/runs/{runId}/results?api-version=7.1&$top={top}&$skip={skip}&outcomes=Failed", + cancellationToken + ); + response.EnsureSuccessStatusCode(); + + var resultsResponse = await response.Content.ReadFromJsonAsync( + AzDoJsonContext.Default.TestResultsResponse, + cancellationToken + ); + if (resultsResponse?.Value is not { Count: > 0 } results) + { + break; + } + + foreach (var result in results) + { + yield return result; + } + + if (resultsResponse.Value.Count < top) + { + break; + } + + skip += top; + } + } + + public async Task> FindBuildsAsync( + string commitSha, + int? definitionId = null, + CancellationToken cancellationToken = default + ) + { + var allBuilds = new List(); + + // We need to query both completed and in-progress builds separately + // because the API defaults to completed builds and may not return in-progress ones + var statusFilters = new[] { "completed", "inProgress", "notStarted" }; + + foreach (var statusFilter in statusFilters) + { + var url = + $"_apis/build/builds?api-version=7.1&$top=500&queryOrder=queueTimeDescending&statusFilter={statusFilter}"; + + if (definitionId.HasValue) + { + url += $"&definitions={definitionId.Value}"; + } + + var response = await _httpClient.GetAsync(url, cancellationToken); + response.EnsureSuccessStatusCode(); + + var buildsResponse = await response.Content.ReadFromJsonAsync( + AzDoJsonContext.Default.BuildsResponse, + cancellationToken + ); + + if (buildsResponse?.Value is { } builds) + { + allBuilds.AddRange(builds); + } + } + + // Filter by commit SHA (case-insensitive prefix match to handle short SHAs) + return allBuilds + .Where(b => + b.SourceVersion != null + && ( + b.SourceVersion.StartsWith(commitSha, StringComparison.OrdinalIgnoreCase) + || commitSha.StartsWith(b.SourceVersion, StringComparison.OrdinalIgnoreCase) + ) + ) + .ToList(); + } + + public async Task> GetActiveBuildsAsync( + int definitionId, + CancellationToken cancellationToken = default + ) + { + // Query specifically for in-progress and not-started builds + var url = + $"_apis/build/builds?api-version=7.1&definitions={definitionId}&statusFilter=inProgress,notStarted&queryOrder=queueTimeDescending&$top=50"; + + var response = await _httpClient.GetAsync(url, cancellationToken); + response.EnsureSuccessStatusCode(); + + var buildsResponse = await response.Content.ReadFromJsonAsync( + AzDoJsonContext.Default.BuildsResponse, + cancellationToken + ); + + return buildsResponse?.Value ?? []; + } + + public async Task> GetRecentBuildsAsync( + int definitionId, + int top = 10, + CancellationToken cancellationToken = default + ) + { + // Query completed builds, ordered by finish time descending + var url = + $"_apis/build/builds?api-version=7.1&definitions={definitionId}&statusFilter=completed&queryOrder=finishTimeDescending&$top={top}"; + + var response = await _httpClient.GetAsync(url, cancellationToken); + response.EnsureSuccessStatusCode(); + + var buildsResponse = await response.Content.ReadFromJsonAsync( + AzDoJsonContext.Default.BuildsResponse, + cancellationToken + ); + + return buildsResponse?.Value ?? []; + } + + public async Task QueueBuildAsync( + int definitionId, + string commitSha, + string? sourceBranch = null, + CancellationToken cancellationToken = default + ) + { + var request = new QueueBuildRequest + { + Definition = new BuildDefinitionReference { Id = definitionId }, + SourceVersion = commitSha, + SourceBranch = sourceBranch, + }; + + var response = await _httpClient.PostAsJsonAsync( + "_apis/build/builds?api-version=7.1", + request, + AzDoJsonContext.Default.QueueBuildRequest, + cancellationToken + ); + + response.EnsureSuccessStatusCode(); + + var build = await response.Content.ReadFromJsonAsync( + AzDoJsonContext.Default.Build, + cancellationToken + ); + return build + ?? throw new InvalidOperationException("Failed to queue build - no response received"); + } + + public void Dispose() + { + _httpClient.Dispose(); + } +} + +// Response wrapper types for AzDO API +internal class TestRunsResponse +{ + public List? Value { get; set; } + public int Count { get; set; } +} + +internal class TestRun +{ + public int Id { get; set; } + public string? Name { get; set; } +} + +internal class TestResultsResponse +{ + public List? Value { get; set; } + public int Count { get; set; } +} + +internal class BuildsResponse +{ + public List? Value { get; set; } + public int Count { get; set; } +} + +internal class QueueBuildRequest +{ + public BuildDefinitionReference? Definition { get; set; } + public string? SourceVersion { get; set; } + public string? SourceBranch { get; set; } +} + +internal class BuildDefinitionReference +{ + public int Id { get; set; } +} + +/// +/// Source-generated JSON serializer context for AOT compatibility. +/// +[JsonSourceGenerationOptions( + PropertyNameCaseInsensitive = true, + Converters = [ + typeof(JsonStringEnumConverter), + typeof(JsonStringEnumConverter), + typeof(JsonStringEnumConverter), + ] +)] +[JsonSerializable(typeof(Build))] +[JsonSerializable(typeof(TestRunsResponse))] +[JsonSerializable(typeof(TestResultsResponse))] +[JsonSerializable(typeof(BuildsResponse))] +[JsonSerializable(typeof(QueueBuildRequest))] +internal partial class AzDoJsonContext : JsonSerializerContext { } diff --git a/src/tools/auto-bisect/src/BisectAlgorithm.cs b/src/tools/auto-bisect/src/BisectAlgorithm.cs new file mode 100644 index 00000000000000..a7f1aa6a262296 --- /dev/null +++ b/src/tools/auto-bisect/src/BisectAlgorithm.cs @@ -0,0 +1,130 @@ +using System; +using System.Collections.Generic; + +namespace AutoBisect; + +/// +/// Result of a single bisect step. +/// +public sealed class BisectStepResult +{ + /// + /// The index of the commit to test next, or -1 if bisect is complete. + /// + public int CommitIndexToTest { get; init; } + + /// + /// Whether the bisect is complete (found the culprit). + /// + public bool IsComplete { get; init; } + + /// + /// The index of the first bad commit, only valid when IsComplete is true. + /// + public int FirstBadCommitIndex { get; init; } +} + +/// +/// Implements binary search (bisect) algorithm for finding the first bad commit. +/// Commits are ordered from oldest (good) to newest (bad). +/// +public sealed class BisectAlgorithm +{ + private readonly IReadOnlyList _commits; + private int _lowIndex; // Inclusive - first index that could be the first bad commit + private int _highIndex; // Inclusive - last index that could be the first bad commit + + /// + /// Creates a new bisect algorithm instance. + /// + /// List of commits ordered from oldest to newest. + public BisectAlgorithm(IReadOnlyList commits) + { + if (commits.Count == 0) + { + throw new ArgumentException("Commits list cannot be empty.", nameof(commits)); + } + + _commits = commits; + _lowIndex = 0; + _highIndex = _commits.Count - 1; + } + + /// + /// Gets the total number of commits in the search space. + /// + public int TotalCommits => _commits.Count; + + /// + /// Gets the number of commits remaining in the search space. + /// + public int RemainingCount => _highIndex - _lowIndex + 1; + + /// + /// Gets the next step in the bisect process. + /// + public BisectStepResult GetNextStep() + { + if (_lowIndex > _highIndex) + { + throw new InvalidOperationException( + $"Bisect invariant violated: _lowIndex ({_lowIndex}) > _highIndex ({_highIndex}). " + + "This indicates a bug in the bisect algorithm or invalid test results."); + } + + if (_lowIndex == _highIndex) + { + // Found the culprit + return new BisectStepResult + { + IsComplete = true, + CommitIndexToTest = -1, + FirstBadCommitIndex = _lowIndex + }; + } + + // Pick the middle commit to test + var midIndex = _lowIndex + (_highIndex - _lowIndex) / 2; + + return new BisectStepResult + { + IsComplete = false, + CommitIndexToTest = midIndex, + FirstBadCommitIndex = -1 + }; + } + + /// + /// Records the result of testing a commit. + /// + /// The index of the commit that was tested. + /// True if the test failed (commit is bad), false if it passed (commit is good). + public void RecordResult(int commitIndex, bool testFailed) + { + if (commitIndex < _lowIndex || commitIndex > _highIndex) + { + throw new ArgumentOutOfRangeException(nameof(commitIndex), + $"Commit index {commitIndex} is outside the current search range [{_lowIndex}, {_highIndex}]."); + } + + if (testFailed) + { + // If test fails at mid, it means mid is bad. The first bad commit could be mid, or any + // commit before mid. So we search [low, mid]. When low >= high, we've found the first + // bad commit. + _highIndex = commitIndex; + } + else + { + // The test passed at this commit, so this commit is good. + // The first bad commit must be after this one. + // Narrow search to [commitIndex + 1, high]. + _lowIndex = commitIndex + 1; + } + } + + /// + /// Gets the commit at the specified index. + /// + public string GetCommit(int index) => _commits[index]; +} diff --git a/src/tools/auto-bisect/src/BuildUtilities.cs b/src/tools/auto-bisect/src/BuildUtilities.cs new file mode 100644 index 00000000000000..8d8335673bd9ec --- /dev/null +++ b/src/tools/auto-bisect/src/BuildUtilities.cs @@ -0,0 +1,118 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using AutoBisect; +using Spectre.Console; + +namespace AutoBisect; + +internal static class BuildUtilities +{ + public static void PrintBuildInfo(Build build) + { + var shortSha = + build.SourceVersion?.Substring(0, Math.Min(12, build.SourceVersion?.Length ?? 0)) + ?? "unknown"; + var statusColor = build.Status == BuildStatus.Completed ? "green" : "yellow"; + var resultColor = + build.Result == BuildResult.Succeeded ? "green" + : build.Result == BuildResult.Failed ? "red" + : "yellow"; + + var table = new Table() + .Border(TableBorder.Rounded) + .BorderColor(Color.Blue) + .Title($"[bold]Build {build.Id}[/]") + .AddColumn("[bold]Property[/]") + .AddColumn("[bold]Value[/]") + .AddRow("Status", $"[{statusColor}]{build.Status}[/]") + .AddRow("Result", $"[{resultColor}]{build.Result}[/]") + .AddRow("Commit", $"[cyan]{shortSha}[/]") + .AddRow("Queued", build.QueueTime?.ToString() ?? "N/A") + .AddRow("Started", build.StartTime?.ToString() ?? "N/A") + .AddRow("Finished", build.FinishTime?.ToString() ?? "N/A"); + + if (build.Links?.Web?.Href is not null) + { + table.AddRow("URL", $"[link]{build.Links.Web.Href}[/]"); + } + + AnsiConsole.Write(table); + } + + public static async Task WaitForBuildAsync( + AzDoClient client, + int buildId, + int pollIntervalSeconds, + string? testName = null, + CancellationToken cancellationToken = default + ) + { + var startTime = DateTime.UtcNow; + Build? build = null; + var earlyExit = false; + + await AnsiConsole + .Status() + .Spinner(Spinner.Known.Dots) + .StartAsync( + $"[yellow]Waiting for build {buildId}...[/]", + async ctx => + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + build = await client.GetBuildAsync(buildId, cancellationToken); + if (build is null) + { + return; + } + + if (build.Status is BuildStatus.Completed) + { + return; + } + + // If a test name is provided, check if it has already failed + if (!string.IsNullOrEmpty(testName)) + { + var hasTestFailed = await client.HasTestFailedAsync( + buildId, + testName, + cancellationToken + ); + if (hasTestFailed) + { + AnsiConsole.MarkupLine( + $"[yellow]⚡[/] Test '{testName.EscapeMarkup()}' has already failed - stopping early" + ); + earlyExit = true; + return; + } + } + + var elapsed = DateTime.UtcNow - startTime; + ctx.Status( + $"[yellow]Waiting for build {buildId}...[/] ({elapsed:hh\\:mm\\:ss} elapsed, status: {build.Status})" + ); + + await Task.Delay( + TimeSpan.FromSeconds(pollIntervalSeconds), + cancellationToken + ); + } + } + ); + + // If we exited early due to test failure, treat the build as if it completed with failure + if (earlyExit && build is not null) + { + // We'll still return the build object, but mark it conceptually as failed + // The calling code will check the actual test results anyway + AnsiConsole.MarkupLine($"[dim]Build {buildId} still running, but test failure detected[/]"); + } + + return build; + } +} diff --git a/src/tools/auto-bisect/src/Commands/BisectCommand.cs b/src/tools/auto-bisect/src/Commands/BisectCommand.cs new file mode 100644 index 00000000000000..fab2e1e04d89e3 --- /dev/null +++ b/src/tools/auto-bisect/src/Commands/BisectCommand.cs @@ -0,0 +1,426 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Spectre.Console; + +namespace AutoBisect.Commands; + +internal static class BisectCommand +{ + public static async Task HandleAsync( + CancellationToken cancellationToken, + string org, + string project, + string pat, + int goodBuildId, + int badBuildId, + string testName, + string repoPath, + bool manual, + int pollInterval + ) + { + using var client = new AzDoClient(org, project, pat); + + // Get build info for good and bad builds + Build? goodBuild = null; + Build? badBuild = null; + await AnsiConsole + .Status() + .Spinner(Spinner.Known.Dots) + .StartAsync( + "[yellow]Fetching build information...[/]", + async ctx => + { + goodBuild = await client.GetBuildAsync(goodBuildId, cancellationToken); + badBuild = await client.GetBuildAsync(badBuildId, cancellationToken); + } + ); + + if (goodBuild == null) + { + Console.Error.WriteLine($"Good build {goodBuildId} not found."); + Environment.ExitCode = 1; + return 1; + } + + if (badBuild == null) + { + Console.Error.WriteLine($"Bad build {badBuildId} not found."); + Environment.ExitCode = 1; + return 1; + } + + var goodCommit = goodBuild.SourceVersion; + var badCommit = badBuild.SourceVersion; + var definitionId = badBuild.Definition?.Id; + var sourceBranch = badBuild.SourceBranch; + + if (string.IsNullOrEmpty(goodCommit) || string.IsNullOrEmpty(badCommit)) + { + Console.Error.WriteLine("Could not determine source commits for builds."); + Environment.ExitCode = 1; + return 1; + } + + if (definitionId == null) + { + Console.Error.WriteLine("Could not determine pipeline definition ID."); + Environment.ExitCode = 1; + return 1; + } + + var configTable = new Table() + .Border(TableBorder.Rounded) + .BorderColor(Color.Blue) + .AddColumn("[bold]Property[/]") + .AddColumn("[bold]Value[/]") + .AddRow("Good commit", $"[green]{goodCommit[..12]}[/] (build {goodBuildId})") + .AddRow("Bad commit", $"[red]{badCommit[..12]}[/] (build {badBuildId})") + .AddRow("Pipeline", $"{badBuild.Definition?.Name} ({definitionId})") + .AddRow("Test", $"[yellow]{testName}[/]") + .AddRow("Mode", manual ? "Manual" : "Auto-queue"); + + AnsiConsole.Write(configTable); + AnsiConsole.WriteLine(); + + // Verify the test actually fails in the bad build + List badFailures = []; + await AnsiConsole + .Status() + .Spinner(Spinner.Known.Dots) + .StartAsync( + "[yellow]Verifying test fails in bad build...[/]", + async ctx => + { + await foreach (var test in client.GetFailedTestsAsync(badBuildId, cancellationToken)) + { + badFailures.Add(test); + } + } + ); + var matchingFailure = badFailures.FirstOrDefault(t => + t.FullyQualifiedName.Contains(testName, StringComparison.OrdinalIgnoreCase) + ); + + if (matchingFailure is null) + { + Console.Error.WriteLine($"Test '{testName}' is not failing in the bad build."); + Console.Error.WriteLine("Available failing tests:"); + foreach (var test in badFailures.Take(10)) + { + Console.Error.WriteLine($" - {test.FullyQualifiedName}"); + } + if (badFailures.Count > 10) + { + Console.Error.WriteLine($" ... and {badFailures.Count - 10} more"); + } + Environment.ExitCode = 1; + return 1; + } + + var fullTestName = matchingFailure.FullyQualifiedName; + AnsiConsole.MarkupLine($"[green]✓[/] Matched test: [cyan]{fullTestName.EscapeMarkup()}[/]"); + + // Verify the test passes (or doesn't exist) in the good build + List goodFailures = []; + await AnsiConsole + .Status() + .Spinner(Spinner.Known.Dots) + .StartAsync( + "[yellow]Verifying test passes in good build...[/]", + async ctx => + { + await foreach (var test in client.GetFailedTestsAsync(goodBuildId, cancellationToken)) + { + goodFailures.Add(test); + } + } + ); + var goodMatchingFailure = goodFailures.FirstOrDefault(t => + t.FullyQualifiedName.Equals(fullTestName, StringComparison.OrdinalIgnoreCase) + ); + + if (goodMatchingFailure is not null) + { + Console.Error.WriteLine( + $"Test '{fullTestName}' is also failing in the good build. Cannot bisect." + ); + Environment.ExitCode = 1; + return 1; + } + + AnsiConsole.MarkupLine("[green]✓[/] Test status verified."); + AnsiConsole.WriteLine(); + + // Get the list of commits between good and bad + List commits = []; + await AnsiConsole + .Status() + .Spinner(Spinner.Known.Dots) + .StartAsync( + "[yellow]Enumerating commits...[/]", + async ctx => + { + commits = ( + await GitHelper.GetCommitRangeAsync( + goodCommit, + badCommit, + repoPath, + cancellationToken + ) + ).ToList(); + } + ); + + if (commits.Count == 0) + { + Console.Error.WriteLine("No commits found between good and bad builds."); + Console.Error.WriteLine( + "Make sure you're in the correct git repository and have fetched all commits." + ); + Environment.ExitCode = 1; + return 1; + } + + var panel = new Panel( + $"[bold]Found {commits.Count} commit(s) to search.[/]\nBisect will require at most [yellow]{Math.Ceiling(Math.Log2(commits.Count + 1))}[/] build(s)." + ) + .BorderColor(Color.Green) + .Header("[bold blue]Bisect Plan[/]"); + AnsiConsole.Write(panel); + AnsiConsole.WriteLine(); + + // Binary search through commits using the BisectAlgorithm + var bisect = new BisectAlgorithm(commits); + + var step = 1; + while (true) + { + var bisectStep = bisect.GetNextStep(); + if (bisectStep.IsComplete) + { + // Found the culprit + var culpritCommit = bisect.GetCommit(bisectStep.FirstBadCommitIndex); + var culpritShortSha = await GitHelper.GetShortShaAsync( + culpritCommit, + repoPath, + cancellationToken + ); + var culpritSubject = await GitHelper.GetCommitSubjectAsync( + culpritCommit, + repoPath, + cancellationToken + ); + + var resultPanel = new Panel( + new Markup( + $"[bold red]First bad commit:[/] [cyan]{culpritShortSha}[/]\n\n" + + $"[dim]{culpritSubject.EscapeMarkup()}[/]\n\n" + + $"[bold]Full SHA:[/] [cyan]{culpritCommit}[/]" + ) + ) + .BorderColor(Color.Red) + .Header("[bold red]🔍 BISECT RESULT[/]"); + + AnsiConsole.Write(resultPanel); + AnsiConsole.WriteLine(); + + // Display summary for filing an issue + var summaryPanel = new Panel( + $"[bold]Test:[/] {fullTestName.EscapeMarkup()}\n" + + $"[bold]First bad commit:[/] {culpritShortSha} ({culpritCommit})\n" + + $"[bold]Good build:[/] {goodBuildId} (commit {goodCommit[..12]})\n" + + $"[bold]Bad build:[/] {badBuildId} (commit {badCommit[..12]})\n" + + $"[bold]Commits searched:[/] {bisect.TotalCommits}" + ) + .BorderColor(Color.Blue) + .Header("[bold blue]📋 Issue Summary[/]"); + + AnsiConsole.Write(summaryPanel); + return 0; + } + + var midCommit = bisect.GetCommit(bisectStep.CommitIndexToTest); + var shortSha = await GitHelper.GetShortShaAsync(midCommit, repoPath, cancellationToken); + var subject = await GitHelper.GetCommitSubjectAsync( + midCommit, + repoPath, + cancellationToken + ); + + AnsiConsole.Write(new Rule($"[bold yellow]Step {step}[/]").RuleStyle("grey")); + AnsiConsole.MarkupLine( + $"Testing commit [cyan]{shortSha}[/] ([yellow]{bisect.RemainingCount}[/] commits remaining)" + ); + AnsiConsole.MarkupLine($"[dim]{subject.EscapeMarkup()}[/]"); + + // Check if we already have a build for this commit + AnsiConsole.MarkupLine( + $"[dim]Searching for builds: definition={definitionId.Value}, commit={midCommit[..12]}...[/]" + ); + var existingBuilds = await client.FindBuildsAsync( + midCommit, + definitionId.Value, + cancellationToken + ); + + // Debug: show what builds we found + if (existingBuilds.Count > 0) + { + AnsiConsole.MarkupLine( + $"[green]✓[/] Found {existingBuilds.Count} existing build(s) for this commit:" + ); + var tree = new Tree("[bold]Builds[/]"); + foreach (var b in existingBuilds) + { + var statusColor = b.Status == BuildStatus.Completed ? "green" : "yellow"; + var resultColor = + b.Result == BuildResult.Succeeded ? "green" + : b.Result == BuildResult.Failed ? "red" + : "yellow"; + tree.AddNode( + $"[{statusColor}]Build {b.Id}[/]: Status=[{statusColor}]{b.Status}[/], Result=[{resultColor}]{b.Result}[/]" + ); + } + AnsiConsole.Write(tree); + } + else + { + AnsiConsole.MarkupLine($"[dim]No existing builds found for commit {shortSha}[/]"); + } + + Build? buildToCheck = existingBuilds.FirstOrDefault(b => + b.Status == BuildStatus.Completed + && ( + b.Result == BuildResult.Succeeded + || b.Result == BuildResult.PartiallySucceeded + || b.Result == BuildResult.Failed + ) + ); + + if (buildToCheck is null) + { + // Check for in-progress builds + var inProgressBuild = existingBuilds.FirstOrDefault(b => + b.Status == BuildStatus.InProgress || b.Status == BuildStatus.NotStarted + ); + + if (inProgressBuild is not null) + { + AnsiConsole.MarkupLine( + $"[yellow]⏳[/] Build {inProgressBuild.Id} is in progress..." + ); + buildToCheck = await BuildUtilities.WaitForBuildAsync( + client, + inProgressBuild.Id, + pollInterval, + fullTestName, + cancellationToken + ); + } + else if (manual) + { + var manualPanel = new Panel( + $"[yellow]No existing build found.[/]\n\nQueue a build for commit: [cyan]{midCommit}[/]\n\nOnce the build completes, re-run this command to continue bisecting." + ) + .BorderColor(Color.Yellow) + .Header("[bold]Manual Action Required[/]"); + AnsiConsole.Write(manualPanel); + return 0; + } + else + { + // Auto-queue a new build + try + { + Build? newBuild = null; + await AnsiConsole + .Status() + .Spinner(Spinner.Known.Dots) + .StartAsync( + "[yellow]Queuing new build...[/]", + async ctx => + { + newBuild = await client.QueueBuildAsync( + definitionId.Value, + midCommit, + sourceBranch, + cancellationToken + ); + } + ); + + if (newBuild is not null) + { + AnsiConsole.MarkupLine( + $"[green]✓[/] Queued build [cyan]{newBuild.Id}[/]" + ); + if (newBuild.Links?.Web?.Href is not null) + { + AnsiConsole.MarkupLine($"[link]{newBuild.Links.Web.Href}[/]"); + } + buildToCheck = await BuildUtilities.WaitForBuildAsync( + client, + newBuild.Id, + pollInterval, + fullTestName, + cancellationToken + ); + } + } + catch (HttpRequestException ex) + { + AnsiConsole.MarkupLine( + $"[red]✗[/] Failed to queue build: {ex.Message.EscapeMarkup()}" + ); + AnsiConsole.MarkupLine( + "[yellow]Try running with --manual mode or queue the build manually.[/]" + ); + Environment.ExitCode = 1; + return 1; + } + } + } + else + { + var resultColor = + buildToCheck.Result == BuildResult.Succeeded ? "green" + : buildToCheck.Result == BuildResult.Failed ? "red" + : "yellow"; + AnsiConsole.MarkupLine( + $"[green]✓[/] Found existing build: [cyan]{buildToCheck.Id}[/] ([{resultColor}]{buildToCheck.Result}[/])" + ); + } + + if (buildToCheck is null) + { + Console.Error.WriteLine(" Build did not complete successfully."); + Environment.ExitCode = 1; + return 1; + } + + // Check test results - build failures can be due to test failures + var testFailed = false; + await foreach (var failure in client.GetFailedTestsAsync(buildToCheck.Id, cancellationToken)) + { + if (failure.FullyQualifiedName.Equals(fullTestName, StringComparison.OrdinalIgnoreCase)) + { + testFailed = true; + break; + } + } + + var resultIcon = testFailed ? "[red]✗ FAILED[/]" : "[green]✓ PASSED[/]"; + AnsiConsole.MarkupLine($"Test result: {resultIcon}"); + AnsiConsole.WriteLine(); + + bisect.RecordResult(bisectStep.CommitIndexToTest, testFailed); + + step++; + } + } +} diff --git a/src/tools/auto-bisect/src/Commands/BuildCommand.cs b/src/tools/auto-bisect/src/Commands/BuildCommand.cs new file mode 100644 index 00000000000000..2004b25d51a5dd --- /dev/null +++ b/src/tools/auto-bisect/src/Commands/BuildCommand.cs @@ -0,0 +1,125 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using AutoBisect; +using Spectre.Console; + +namespace AutoBisect.Commands; + +internal static class BuildCommand +{ + public static async Task HandleAsync(string org, string project, string pat, int buildId) + { + using var client = new AzDoClient(org, project, pat); + Build? build = null; + await AnsiConsole + .Status() + .Spinner(Spinner.Known.Dots) + .StartAsync( + "[yellow]Fetching build information...[/]", + async ctx => + { + build = await client.GetBuildAsync(buildId); + } + ); + + if (build == null) + { + AnsiConsole.MarkupLine($"[red]✗[/] Build {buildId} not found."); + Environment.ExitCode = 1; + return; + } + + var statusColor = build.Status == BuildStatus.Completed ? "green" : "yellow"; + var resultColor = + build.Result == BuildResult.Succeeded ? "green" + : build.Result == BuildResult.Failed ? "red" + : "yellow"; + + var table = new Table() + .Border(TableBorder.Rounded) + .BorderColor(Color.Blue) + .AddColumn("[bold]Property[/]") + .AddColumn("[bold]Value[/]") + .AddRow("Build ID", $"[cyan]{build.Id}[/]") + .AddRow("Build Number", build.BuildNumber ?? "N/A") + .AddRow("Status", $"[{statusColor}]{build.Status}[/]") + .AddRow("Result", $"[{resultColor}]{build.Result}[/]") + .AddRow("Source Version", $"[cyan]{build.SourceVersion?[..12]}[/]") + .AddRow("Source Branch", build.SourceBranch ?? "N/A") + .AddRow("Definition", $"{build.Definition?.Name} ({build.Definition?.Id})") + .AddRow("Queue Time", build.QueueTime?.ToString() ?? "N/A") + .AddRow("Start Time", build.StartTime?.ToString() ?? "N/A") + .AddRow("Finish Time", build.FinishTime?.ToString() ?? "N/A"); + + if (build.Links?.Web?.Href != null) + { + table.AddRow("Web URL", $"[link]{build.Links.Web.Href}[/]"); + } + + AnsiConsole.Write(table); + + // Fetch and display test failures if the build has completed + if (build.Status == BuildStatus.Completed) + { + AnsiConsole.WriteLine(); + List failedTests = []; + await AnsiConsole + .Status() + .Spinner(Spinner.Known.Dots) + .StartAsync( + "[yellow]Fetching failed tests...[/]", + async ctx => + { + try + { + await foreach (var test in client.GetFailedTestsAsync(buildId)) + { + failedTests.Add(test); + } + } + catch (HttpRequestException ex) + when (ex.StatusCode == System.Net.HttpStatusCode.Unauthorized) + { + // Will handle below + } + } + ); + + if (failedTests.Count > 0) + { + var testTable = new Table() + .Border(TableBorder.Rounded) + .BorderColor(Color.Red) + .AddColumn("[bold red]Failed Tests[/]") + .AddColumn("[bold]Error Message[/]"); + + foreach (var test in failedTests.OrderBy(t => t.FullyQualifiedName)) + { + var errorMsg = ""; + if (!string.IsNullOrWhiteSpace(test.ErrorMessage)) + { + var firstLine = test.ErrorMessage.Split('\n')[0].Trim(); + if (firstLine.Length > 100) + { + firstLine = firstLine.Substring(0, 97) + "..."; + } + errorMsg = $"[dim]{firstLine.EscapeMarkup()}[/]"; + } + testTable.AddRow( + $"[red]✗[/] {test.FullyQualifiedName.EscapeMarkup()}", + errorMsg + ); + } + + AnsiConsole.Write(testTable); + } + else if (failedTests.Count == 0) + { + AnsiConsole.MarkupLine("[green]✓[/] No failed tests found."); + } + } + } +} diff --git a/src/tools/auto-bisect/src/Commands/DiffCommand.cs b/src/tools/auto-bisect/src/Commands/DiffCommand.cs new file mode 100644 index 00000000000000..02c858d4053819 --- /dev/null +++ b/src/tools/auto-bisect/src/Commands/DiffCommand.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AutoBisect; +using Spectre.Console; + +namespace AutoBisect.Commands; + +internal static class DiffCommand +{ + public static async Task HandleAsync( + string org, + string project, + string pat, + int goodBuildId, + int badBuildId + ) + { + using var client = new AzDoClient(org, project, pat); + + List goodFailures = []; + List badFailures = []; + + await AnsiConsole + .Status() + .Spinner(Spinner.Known.Dots) + .StartAsync( + "[yellow]Fetching test results...[/]", + async ctx => + { + ctx.Status( + $"[yellow]Fetching failed tests for build {goodBuildId} (good)...[/]" + ); + await foreach (var test in client.GetFailedTestsAsync(goodBuildId)) + { + goodFailures.Add(test); + } + + ctx.Status($"[yellow]Fetching failed tests for build {badBuildId} (bad)...[/]"); + await foreach (var test in client.GetFailedTestsAsync(badBuildId)) + { + badFailures.Add(test); + } + } + ); + + var diff = TestDiffer.ComputeDiff(goodFailures, badFailures); + + var newFailuresTable = new Table() + .Border(TableBorder.Rounded) + .BorderColor(Color.Red) + .Title($"[bold red]New Failures ({diff.NewFailures.Count})[/]") + .AddColumn("[bold]Test Name[/]"); + + if (diff.NewFailures.Count > 0) + { + foreach (var test in diff.NewFailures) + { + newFailuresTable.AddRow($"[red]✗[/] {test.EscapeMarkup()}"); + } + } + else + { + newFailuresTable.AddRow("[dim]No new failures[/]"); + } + + AnsiConsole.Write(newFailuresTable); + AnsiConsole.WriteLine(); + + var consistentFailuresTable = new Table() + .Border(TableBorder.Rounded) + .BorderColor(Color.Yellow) + .Title($"[bold yellow]Consistent Failures ({diff.ConsistentFailures.Count})[/]") + .AddColumn("[bold]Test Name[/]"); + + if (diff.ConsistentFailures.Count > 0) + { + foreach (var test in diff.ConsistentFailures) + { + consistentFailuresTable.AddRow($"[yellow]─[/] {test.EscapeMarkup()}"); + } + } + else + { + consistentFailuresTable.AddRow("[dim]No consistent failures[/]"); + } + + AnsiConsole.Write(consistentFailuresTable); + } +} diff --git a/src/tools/auto-bisect/src/Commands/QueuedCommand.cs b/src/tools/auto-bisect/src/Commands/QueuedCommand.cs new file mode 100644 index 00000000000000..e619608bff224c --- /dev/null +++ b/src/tools/auto-bisect/src/Commands/QueuedCommand.cs @@ -0,0 +1,54 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using AutoBisect; + +namespace AutoBisect.Commands; + +internal static class QueuedCommand +{ + public static async Task HandleAsync( + string org, + string project, + string pat, + int definitionId, + bool showAll + ) + { + using var client = new AzDoClient(org, project, pat); + + Console.WriteLine($"Fetching builds for definition {definitionId}..."); + Console.WriteLine(); + + var activeBuilds = await client.GetActiveBuildsAsync(definitionId); + + if (activeBuilds.Count > 0) + { + Console.WriteLine($"Active builds ({activeBuilds.Count}):"); + Console.WriteLine(); + foreach (var build in activeBuilds.OrderByDescending(b => b.QueueTime)) + { + BuildUtilities.PrintBuildInfo(build); + } + } + else + { + Console.WriteLine("No active builds."); + Console.WriteLine(); + } + + if (showAll) + { + var recentBuilds = await client.GetRecentBuildsAsync(definitionId, top: 10); + if (recentBuilds.Count > 0) + { + Console.WriteLine($"Recent completed builds ({recentBuilds.Count}):"); + Console.WriteLine(); + foreach (var build in recentBuilds) + { + BuildUtilities.PrintBuildInfo(build); + } + } + } + } +} diff --git a/src/tools/auto-bisect/src/Commands/TestsCommand.cs b/src/tools/auto-bisect/src/Commands/TestsCommand.cs new file mode 100644 index 00000000000000..ecfac28bbe4ef9 --- /dev/null +++ b/src/tools/auto-bisect/src/Commands/TestsCommand.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AutoBisect; +using Spectre.Console; + +namespace AutoBisect.Commands; + +internal static class TestsCommand +{ + public static async Task HandleAsync(string org, string project, string pat, int buildId) + { + using var client = new AzDoClient(org, project, pat); + List results = []; + await AnsiConsole + .Status() + .Spinner(Spinner.Known.Dots) + .StartAsync( + $"[yellow]Fetching failed tests for build {buildId}...[/]", + async ctx => + { + await foreach (var test in client.GetFailedTestsAsync(buildId)) + { + results.Add(test); + } + } + ); + + if (results.Count == 0) + { + AnsiConsole.MarkupLine($"[green]✓[/] No failed tests found in build {buildId}."); + return; + } + + var table = new Table() + .Border(TableBorder.Rounded) + .BorderColor(Color.Red) + .Title($"[bold red]Failed Tests ({results.Count})[/]") + .AddColumn("[bold]Test Name[/]") + .AddColumn("[bold]Error Message[/]"); + + foreach (var result in results.OrderBy(r => r.FullyQualifiedName)) + { + var errorMsg = ""; + if (!string.IsNullOrWhiteSpace(result.ErrorMessage)) + { + var firstLine = result.ErrorMessage.Split('\n')[0].Trim(); + if (firstLine.Length > 100) + { + firstLine = firstLine.Substring(0, 97) + "..."; + } + errorMsg = $"[dim]{firstLine.EscapeMarkup()}[/]"; + } + table.AddRow($"[red]✗[/] {result.FullyQualifiedName.EscapeMarkup()}", errorMsg); + } + + AnsiConsole.Write(table); + } +} diff --git a/src/tools/auto-bisect/src/GitHelper.cs b/src/tools/auto-bisect/src/GitHelper.cs new file mode 100644 index 00000000000000..66e4102fa21722 --- /dev/null +++ b/src/tools/auto-bisect/src/GitHelper.cs @@ -0,0 +1,166 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace AutoBisect; + +/// +/// Helper class for Git operations. +/// +public static class GitHelper +{ + /// + /// Gets the list of commit SHAs between two commits (exclusive of good, inclusive of bad). + /// Returns commits in chronological order (oldest first). + /// + public static async Task> GetCommitRangeAsync( + string goodCommit, + string badCommit, + string? workingDirectory = null, + CancellationToken cancellationToken = default + ) + { + // First, verify both commits exist locally + if (!await ValidateCommitAsync(goodCommit, workingDirectory, cancellationToken)) + { + throw new InvalidOperationException( + $"Commit {goodCommit} not found locally. Try running: git fetch origin {goodCommit}" + ); + } + + if (!await ValidateCommitAsync(badCommit, workingDirectory, cancellationToken)) + { + throw new InvalidOperationException( + $"Commit {badCommit} not found locally. Try running: git fetch origin {badCommit}" + ); + } + + // git rev-list returns commits in reverse chronological order (newest first) + // We use --ancestry-path to only get commits on the path from good to bad + string result; + try + { + result = await RunGitCommandAsync( + $"rev-list --ancestry-path {goodCommit}..{badCommit}", + workingDirectory, + cancellationToken + ); + } + catch (InvalidOperationException) + { + // --ancestry-path may fail if commits aren't on a direct path + // Fall back to simple range + result = await RunGitCommandAsync( + $"rev-list {goodCommit}..{badCommit}", + workingDirectory, + cancellationToken + ); + } + + var commits = new List(); + foreach (var line in result.Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + var sha = line.Trim(); + if (!string.IsNullOrEmpty(sha)) + { + commits.Add(sha); + } + } + + // Reverse to get chronological order (oldest first) + commits.Reverse(); + return commits; + } + + /// + /// Gets the short SHA for a commit. + /// + public static async Task GetShortShaAsync( + string commit, + string? workingDirectory = null, + CancellationToken cancellationToken = default + ) + { + var result = await RunGitCommandAsync( + $"rev-parse --short {commit}", + workingDirectory, + cancellationToken + ); + return result.Trim(); + } + + /// + /// Gets the commit message subject (first line) for a commit. + /// + public static async Task GetCommitSubjectAsync( + string commit, + string? workingDirectory = null, + CancellationToken cancellationToken = default + ) + { + var result = await RunGitCommandAsync( + $"log -1 --format=%s {commit}", + workingDirectory, + cancellationToken + ); + return result.Trim(); + } + + /// + /// Validates that a commit SHA exists in the repository. + /// + public static async Task ValidateCommitAsync( + string commit, + string? workingDirectory = null, + CancellationToken cancellationToken = default + ) + { + try + { + await RunGitCommandAsync($"cat-file -t {commit}", workingDirectory, cancellationToken); + return true; + } + catch (InvalidOperationException) + { + return false; + } + } + + private static async Task RunGitCommandAsync( + string arguments, + string? workingDirectory, + CancellationToken cancellationToken + ) + { + using var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = "git", + Arguments = arguments, + WorkingDirectory = workingDirectory ?? Environment.CurrentDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }, + }; + + process.Start(); + + var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken); + var errorTask = process.StandardError.ReadToEndAsync(cancellationToken); + + await process.WaitForExitAsync(cancellationToken); + + if (process.ExitCode != 0) + { + var error = await errorTask; + throw new InvalidOperationException($"Git command failed: {error}"); + } + + return await outputTask; + } +} diff --git a/src/tools/auto-bisect/src/Models.cs b/src/tools/auto-bisect/src/Models.cs new file mode 100644 index 00000000000000..b9ef2e786c1025 --- /dev/null +++ b/src/tools/auto-bisect/src/Models.cs @@ -0,0 +1,159 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; + +namespace AutoBisect; + +/// +/// Represents an Azure DevOps build. +/// +public class Build +{ + public int Id { get; set; } + public string? BuildNumber { get; set; } + public BuildStatus Status { get; set; } + public BuildResult? Result { get; set; } + public string? SourceVersion { get; set; } + public string? SourceBranch { get; set; } + public DateTime? QueueTime { get; set; } + public DateTime? StartTime { get; set; } + public DateTime? FinishTime { get; set; } + public BuildDefinition? Definition { get; set; } + public string? Url { get; set; } + + [JsonPropertyName("_links")] + public BuildLinks? Links { get; set; } +} + +public class BuildDefinition +{ + public int Id { get; set; } + public string? Name { get; set; } +} + +public class BuildLinks +{ + public Link? Web { get; set; } +} + +public class Link +{ + public string? Href { get; set; } +} + +public enum BuildStatus +{ + None, + InProgress, + Completed, + Cancelling, + Postponed, + NotStarted, + All, +} + +public enum BuildResult +{ + None, + Succeeded, + PartiallySucceeded, + Failed, + Canceled, +} + +/// +/// Represents a test result from Azure DevOps. +/// +public class TestResult +{ + public int Id { get; set; } + public string? TestCaseTitle { get; set; } + public string? AutomatedTestName { get; set; } + public string? AutomatedTestStorage { get; set; } + public TestOutcome Outcome { get; set; } + public string? ErrorMessage { get; set; } + public string? StackTrace { get; set; } + public double DurationInMs { get; set; } + + /// + /// Gets the fully qualified test name. + /// + public string FullyQualifiedName => AutomatedTestName ?? TestCaseTitle ?? $"Unknown-{Id}"; +} + +public enum TestOutcome +{ + Unspecified, + None, + Passed, + Failed, + Inconclusive, + Timeout, + Aborted, + Blocked, + NotExecuted, + Warning, + Error, + NotApplicable, + Paused, + InProgress, + NotImpacted, +} + +/// +/// Represents the diff between test failures from two builds. +/// +public class TestFailureDiff +{ + /// + /// Tests that failed in the bad build but not in the good build. + /// + public required List NewFailures { get; init; } + + /// + /// Tests that failed in both builds. + /// + public required List ConsistentFailures { get; init; } +} + +/// +/// Utility class for computing test failure diffs. +/// +public static class TestDiffer +{ + /// + /// Computes the diff between failed tests from two builds. + /// + public static TestFailureDiff ComputeDiff( + IEnumerable goodBuildFailures, + IEnumerable badBuildFailures + ) + { + var goodFailures = goodBuildFailures + .Select(t => t.FullyQualifiedName) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + var badFailures = badBuildFailures + .Select(t => t.FullyQualifiedName) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + // New failures: failed in bad, but not failed in good + var newFailures = badFailures + .Where(t => !goodFailures.Contains(t)) + .OrderBy(t => t) + .ToList(); + + // Consistent failures: failed in both + var consistentFailures = badFailures + .Where(t => goodFailures.Contains(t)) + .OrderBy(t => t) + .ToList(); + + return new TestFailureDiff + { + NewFailures = newFailures, + ConsistentFailures = consistentFailures, + }; + } +} diff --git a/src/tools/auto-bisect/src/Program.cs b/src/tools/auto-bisect/src/Program.cs new file mode 100644 index 00000000000000..5150d323edd0e5 --- /dev/null +++ b/src/tools/auto-bisect/src/Program.cs @@ -0,0 +1,231 @@ +using System; +using System.CommandLine; +using System.CommandLine.Parsing; +using System.Threading.Tasks; +using AutoBisect.Commands; + +var rootCommand = new RootCommand("Azure DevOps bisect tool for finding test regressions"); + +// Global options +var orgOption = new Option("--organization") +{ + Description = "Azure DevOps organization name", + Required = true, +}; +orgOption.Aliases.Add("-o"); + +var projectOption = new Option("--project") +{ + Description = "Azure DevOps project name", + Required = true, +}; +projectOption.Aliases.Add("-p"); + +var patOption = new Option("--pat") +{ + Description = "Personal Access Token (or set AZDO_PAT environment variable)", + DefaultValueFactory = _ => Environment.GetEnvironmentVariable("AZDO_PAT") ?? "", +}; +patOption.Validators.Add(result => +{ + var value = result.GetValueOrDefault(); + if (string.IsNullOrWhiteSpace(value)) + { + result.AddError("PAT is required. Use --pat or set AZDO_PAT environment variable."); + } +}); + +orgOption.Recursive = true; +projectOption.Recursive = true; +patOption.Recursive = true; + +rootCommand.Options.Add(orgOption); +rootCommand.Options.Add(projectOption); +rootCommand.Options.Add(patOption); + +// Add commands +rootCommand.Subcommands.Add(CreateBuildCommand()); +rootCommand.Subcommands.Add(CreateTestsCommand()); +rootCommand.Subcommands.Add(CreateDiffCommand()); +rootCommand.Subcommands.Add(CreateQueuedCommand()); +rootCommand.Subcommands.Add(CreateBisectCommand()); + +return rootCommand.Parse(args).InvokeAsync().GetAwaiter().GetResult(); + +Command CreateBuildCommand() +{ + var command = new Command("build", "Get information about a build"); + var buildIdArgument = new Argument("build-id") { Description = "The build ID to fetch" }; + + command.Arguments.Add(buildIdArgument); + command.SetAction(parseResult => + { + var buildId = parseResult.GetValue(buildIdArgument); + return BuildCommand.HandleAsync( + parseResult.GetValue(orgOption)!, + parseResult.GetValue(projectOption)!, + parseResult.GetValue(patOption)!, + buildId); + }); + + return command; +} + +Command CreateTestsCommand() +{ + var command = new Command("tests", "Get failed test results for a build"); + var buildIdArgument = new Argument("build-id") + { + Description = "The build ID to fetch failed test results for", + }; + + command.Arguments.Add(buildIdArgument); + command.SetAction(parseResult => + { + var buildId = parseResult.GetValue(buildIdArgument); + return TestsCommand.HandleAsync( + parseResult.GetValue(orgOption)!, + parseResult.GetValue(projectOption)!, + parseResult.GetValue(patOption)!, + buildId); + }); + + return command; +} + +Command CreateDiffCommand() +{ + var command = new Command("diff", "Compare test results between two builds"); + + var goodBuildOption = new Option("--good") + { + Description = "Build ID of the known good build", + Required = true, + }; + goodBuildOption.Aliases.Add("-g"); + + var badBuildOption = new Option("--bad") + { + Description = "Build ID of the known bad build", + Required = true, + }; + badBuildOption.Aliases.Add("-b"); + + command.Options.Add(goodBuildOption); + command.Options.Add(badBuildOption); + command.SetAction(parseResult => + { + return DiffCommand.HandleAsync( + parseResult.GetValue(orgOption)!, + parseResult.GetValue(projectOption)!, + parseResult.GetValue(patOption)!, + parseResult.GetValue(goodBuildOption), + parseResult.GetValue(badBuildOption)); + }); + + return command; +} + +Command CreateQueuedCommand() +{ + var command = new Command("queued", "Show active (running/queued) builds for a definition"); + + var definitionIdOption = new Option("--definition") + { + Description = "Build definition ID to filter by", + Required = true, + }; + definitionIdOption.Aliases.Add("-d"); + + var showAllOption = new Option("--all") + { + Description = "Show recent completed builds as well", + }; + showAllOption.Aliases.Add("-a"); + + command.Options.Add(definitionIdOption); + command.Options.Add(showAllOption); + command.SetAction(parseResult => + { + return QueuedCommand.HandleAsync( + parseResult.GetValue(orgOption)!, + parseResult.GetValue(projectOption)!, + parseResult.GetValue(patOption)!, + parseResult.GetValue(definitionIdOption), + parseResult.GetValue(showAllOption)); + }); + + return command; +} + +Command CreateBisectCommand() +{ + var command = new Command("bisect", "Find the commit that introduced a test failure"); + + var goodBuildOption = new Option("--good") + { + Description = "Build ID of the known good build (test passes)", + Required = true, + }; + goodBuildOption.Aliases.Add("-g"); + + var badBuildOption = new Option("--bad") + { + Description = "Build ID of the known bad build (test fails)", + Required = true, + }; + badBuildOption.Aliases.Add("-b"); + + var testNameOption = new Option("--test") + { + Description = "Fully qualified name of the test to track (or substring to match)", + Required = true, + }; + testNameOption.Aliases.Add("-t"); + + var repoPathOption = new Option("--repo") + { + Description = "Path to the git repository (defaults to current directory)", + DefaultValueFactory = _ => Environment.CurrentDirectory, + }; + repoPathOption.Aliases.Add("-r"); + + var manualOption = new Option("--manual") + { + Description = "Don't auto-queue builds; just report what needs to be done", + }; + manualOption.Aliases.Add("-m"); + + var pollIntervalOption = new Option("--poll-interval") + { + Description = "Seconds between polling for build completion", + DefaultValueFactory = _ => 300, + }; + + command.Options.Add(goodBuildOption); + command.Options.Add(badBuildOption); + command.Options.Add(testNameOption); + command.Options.Add(repoPathOption); + command.Options.Add(manualOption); + command.Options.Add(pollIntervalOption); + + command.SetAction( + (parseResult, cancellationToken) => + { + return BisectCommand.HandleAsync( + cancellationToken, + parseResult.GetValue(orgOption)!, + parseResult.GetValue(projectOption)!, + parseResult.GetValue(patOption)!, + parseResult.GetValue(goodBuildOption), + parseResult.GetValue(badBuildOption), + parseResult.GetValue(testNameOption)!, + parseResult.GetValue(repoPathOption)!, + parseResult.GetValue(manualOption), + parseResult.GetValue(pollIntervalOption) + ); + } + ); + + return command; +} diff --git a/src/tools/auto-bisect/src/auto-bisect.csproj b/src/tools/auto-bisect/src/auto-bisect.csproj new file mode 100644 index 00000000000000..d7cc5d8d5e1e6b --- /dev/null +++ b/src/tools/auto-bisect/src/auto-bisect.csproj @@ -0,0 +1,16 @@ + + + Exe + net10.0 + enable + AutoBisect + true + + + + + + + + + diff --git a/src/tools/auto-bisect/tests/AutoBisect.Tests.csproj b/src/tools/auto-bisect/tests/AutoBisect.Tests.csproj new file mode 100644 index 00000000000000..5b8123b44c8a8a --- /dev/null +++ b/src/tools/auto-bisect/tests/AutoBisect.Tests.csproj @@ -0,0 +1,20 @@ + + + net10.0 + enable + false + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + diff --git a/src/tools/auto-bisect/tests/BisectAlgorithmTests.cs b/src/tools/auto-bisect/tests/BisectAlgorithmTests.cs new file mode 100644 index 00000000000000..fe509112f5d9c2 --- /dev/null +++ b/src/tools/auto-bisect/tests/BisectAlgorithmTests.cs @@ -0,0 +1,330 @@ +using System; +using Xunit; + +namespace AutoBisect.Tests; + +public class BisectAlgorithmTests +{ + [Fact] + public void Constructor_EmptyCommits_ThrowsArgumentException() + { + Assert.Throws(() => new BisectAlgorithm(Array.Empty())); + } + + [Fact] + public void Constructor_ValidCommits_SetsRemainingCount() + { + var commits = new[] { "a", "b", "c", "d", "e" }; + var algo = new BisectAlgorithm(commits); + + Assert.Equal(5, algo.RemainingCount); + } + + [Fact] + public void SingleCommit_ImmediatelyComplete() + { + var algo = new BisectAlgorithm(new[] { "commit1" }); + + var step = algo.GetNextStep(); + + Assert.True(step.IsComplete); + Assert.Equal(0, step.FirstBadCommitIndex); + Assert.Equal(1, algo.RemainingCount); + } + + [Fact] + public void TwoCommits_FirstIsBad_FindsFirst() + { + // Commits: [good, bad] but if we test the first and it's bad, first is the culprit + // Actually in bisect, we assume the range is [potentially_bad..potentially_bad] + // Let's think: commits[0] could be first bad, commits[1] could be first bad + // Test commits[0]: if bad, commits[0] is the first bad + var algo = new BisectAlgorithm(new[] { "commit0", "commit1" }); + + // With 2 commits, midIndex = 0 + (1-0)/2 = 0 + var step1 = algo.GetNextStep(); + Assert.False(step1.IsComplete); + Assert.Equal(0, step1.CommitIndexToTest); + Assert.Equal(2, algo.RemainingCount); + + // Test at index 0 fails (bad commit) + algo.RecordResult(0, testFailed: true); + + // high = 0, low = 0, so should be complete + var step2 = algo.GetNextStep(); + Assert.True(step2.IsComplete); + Assert.Equal(0, step2.FirstBadCommitIndex); + } + + [Fact] + public void TwoCommits_SecondIsBad_FindsSecond() + { + var algo = new BisectAlgorithm(new[] { "commit0", "commit1" }); + + // With 2 commits, midIndex = 0 + (1-0)/2 = 0 + var step1 = algo.GetNextStep(); + Assert.False(step1.IsComplete); + Assert.Equal(0, step1.CommitIndexToTest); + + // Test at index 0 passes (good commit) + algo.RecordResult(0, testFailed: false); + + // low = 1, high = 1, so should be complete + var step2 = algo.GetNextStep(); + Assert.True(step2.IsComplete); + Assert.Equal(1, step2.FirstBadCommitIndex); + } + + [Fact] + public void ThreeCommits_MiddleIsBad_FindsMiddle() + { + // Commits: [good, BAD, bad] + // First bad is at index 1 + var algo = new BisectAlgorithm(new[] { "commit0", "commit1", "commit2" }); + + // midIndex = 0 + (2-0)/2 = 1 + var step1 = algo.GetNextStep(); + Assert.False(step1.IsComplete); + Assert.Equal(1, step1.CommitIndexToTest); + Assert.Equal(3, algo.RemainingCount); + + // Test at index 1 fails + algo.RecordResult(1, testFailed: true); + // high = 1, low = 0 + + // midIndex = 0 + (1-0)/2 = 0 + var step2 = algo.GetNextStep(); + Assert.False(step2.IsComplete); + Assert.Equal(0, step2.CommitIndexToTest); + + // Test at index 0 passes + algo.RecordResult(0, testFailed: false); + // low = 1, high = 1 + + var step3 = algo.GetNextStep(); + Assert.True(step3.IsComplete); + Assert.Equal(1, step3.FirstBadCommitIndex); + } + + [Fact] + public void ThreeCommits_LastIsBad_FindsLast() + { + // Commits: [good, good, BAD] + var algo = new BisectAlgorithm(new[] { "commit0", "commit1", "commit2" }); + + // midIndex = 1 + var step1 = algo.GetNextStep(); + Assert.Equal(1, step1.CommitIndexToTest); + + // Test at index 1 passes + algo.RecordResult(1, testFailed: false); + // low = 2, high = 2 + + var step2 = algo.GetNextStep(); + Assert.True(step2.IsComplete); + Assert.Equal(2, step2.FirstBadCommitIndex); + } + + [Fact] + public void ThreeCommits_FirstIsBad_FindsFirst() + { + // Commits: [BAD, bad, bad] + var algo = new BisectAlgorithm(new[] { "commit0", "commit1", "commit2" }); + + // midIndex = 1 + var step1 = algo.GetNextStep(); + Assert.Equal(1, step1.CommitIndexToTest); + + // Test at index 1 fails + algo.RecordResult(1, testFailed: true); + // high = 1, low = 0 + + // midIndex = 0 + var step2 = algo.GetNextStep(); + Assert.Equal(0, step2.CommitIndexToTest); + + // Test at index 0 fails + algo.RecordResult(0, testFailed: true); + // high = 0, low = 0 + + var step3 = algo.GetNextStep(); + Assert.True(step3.IsComplete); + Assert.Equal(0, step3.FirstBadCommitIndex); + } + + [Fact] + public void TenCommits_BadAtIndex7_FindsCorrectly() + { + // Commits 0-6 are good, commits 7-9 are bad + // First bad is at index 7 + var commits = new[] { "c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9" }; + var algo = new BisectAlgorithm(commits); + + var result = RunBisect(algo, firstBadIndex: 7); + + Assert.Equal(7, result); + } + + [Fact] + public void TenCommits_BadAtIndex0_FindsCorrectly() + { + var commits = new[] { "c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9" }; + var algo = new BisectAlgorithm(commits); + + var result = RunBisect(algo, firstBadIndex: 0); + + Assert.Equal(0, result); + } + + [Fact] + public void TenCommits_BadAtIndex9_FindsCorrectly() + { + var commits = new[] { "c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9" }; + var algo = new BisectAlgorithm(commits); + + var result = RunBisect(algo, firstBadIndex: 9); + + Assert.Equal(9, result); + } + + [Fact] + public void HundredCommits_BadAtIndex50_FindsInLogNSteps() + { + var commits = new string[100]; + for (int i = 0; i < 100; i++) + { + commits[i] = $"commit{i}"; + } + + var algo = new BisectAlgorithm(commits); + var steps = 0; + var firstBadIndex = 50; + + while (true) + { + var step = algo.GetNextStep(); + if (step.IsComplete) + { + Assert.Equal(firstBadIndex, step.FirstBadCommitIndex); + break; + } + + steps++; + var testFailed = step.CommitIndexToTest >= firstBadIndex; + algo.RecordResult(step.CommitIndexToTest, testFailed); + } + + // Should complete in at most ceil(log2(100)) = 7 steps + Assert.True(steps <= 7, $"Expected at most 7 steps, but took {steps}"); + } + + [Fact] + public void HundredTwoCommits_BadAtMiddle_MatchesExpectedBehavior() + { + // This test verifies the specific scenario from the bug report + // 102 commits, testing at index 51 should leave 51 commits when test passes + var commits = new string[102]; + for (int i = 0; i < 102; i++) + { + commits[i] = $"commit{i}"; + } + + var algo = new BisectAlgorithm(commits); + + // First step: should test at index 51 (102/2 = 51) + // Actually with low=0, high=101: mid = 0 + (101-0)/2 = 50 + var step1 = algo.GetNextStep(); + Assert.False(step1.IsComplete); + Assert.Equal(50, step1.CommitIndexToTest); // (0 + 101) / 2 = 50 + + // If test passes at index 50, remaining should be indices 51-101 = 51 commits + algo.RecordResult(50, testFailed: false); + Assert.Equal(51, algo.RemainingCount); + + // Next step should test the middle of the remaining range + var step2 = algo.GetNextStep(); + Assert.False(step2.IsComplete); + // low=51, high=101, mid = 51 + (101-51)/2 = 51 + 25 = 76 + Assert.Equal(76, step2.CommitIndexToTest); + } + + [Fact] + public void NoInfiniteLoop_TwoCommits_AllFailScenarios() + { + // This specifically tests the bug that caused infinite loops + // Test all possible scenarios with 2 commits + + // Scenario 1: First is bad + var algo1 = new BisectAlgorithm(new[] { "a", "b" }); + var result1 = RunBisectWithMaxSteps(algo1, firstBadIndex: 0, maxSteps: 10); + Assert.Equal(0, result1); + + // Scenario 2: Second is bad (first is good) + var algo2 = new BisectAlgorithm(new[] { "a", "b" }); + var result2 = RunBisectWithMaxSteps(algo2, firstBadIndex: 1, maxSteps: 10); + Assert.Equal(1, result2); + } + + [Fact] + public void NoInfiniteLoop_ConsecutiveBadCommits() + { + // Test scenario where multiple consecutive commits are bad + // Should find the FIRST bad commit + var commits = new[] { "good1", "good2", "BAD1", "bad2", "bad3" }; + var algo = new BisectAlgorithm(commits); + + var result = RunBisectWithMaxSteps(algo, firstBadIndex: 2, maxSteps: 10); + Assert.Equal(2, result); + } + + [Fact] + public void RecordResult_OutOfRange_ThrowsException() + { + var algo = new BisectAlgorithm(new[] { "a", "b", "c" }); + + Assert.Throws(() => algo.RecordResult(-1, true)); + Assert.Throws(() => algo.RecordResult(3, true)); + } + + /// + /// Runs the bisect algorithm to completion and returns the index of the first bad commit. + /// + private static int RunBisect(BisectAlgorithm algo, int firstBadIndex) + { + while (true) + { + var step = algo.GetNextStep(); + if (step.IsComplete) + { + return step.FirstBadCommitIndex; + } + + var testFailed = step.CommitIndexToTest >= firstBadIndex; + algo.RecordResult(step.CommitIndexToTest, testFailed); + } + } + + /// + /// Runs the bisect algorithm with a maximum number of steps to prevent infinite loops. + /// Returns -1 if max steps exceeded. + /// + private static int RunBisectWithMaxSteps(BisectAlgorithm algo, int firstBadIndex, int maxSteps) + { + var steps = 0; + while (steps < maxSteps) + { + var step = algo.GetNextStep(); + if (step.IsComplete) + { + return step.FirstBadCommitIndex; + } + + steps++; + var testFailed = step.CommitIndexToTest >= firstBadIndex; + algo.RecordResult(step.CommitIndexToTest, testFailed); + } + + Assert.Fail($"Bisect did not complete within {maxSteps} steps - possible infinite loop!"); + return -1; + } +} diff --git a/src/tools/auto-bisect/tests/ModelsTests.cs b/src/tools/auto-bisect/tests/ModelsTests.cs new file mode 100644 index 00000000000000..8f38c286e30fa9 --- /dev/null +++ b/src/tools/auto-bisect/tests/ModelsTests.cs @@ -0,0 +1,82 @@ +using System.Collections.Generic; +using AutoBisect; +using Xunit; + +namespace AutoBisect.Tests; + +public class ModelsTests +{ + [Fact] + public void TestResult_FullyQualifiedName_UsesAutomatedTestName() + { + // Arrange + var result = new TestResult + { + Id = 1, + AutomatedTestName = "MyNamespace.MyClass.MyTest", + TestCaseTitle = "My Test Title", + }; + + // Assert + Assert.Equal("MyNamespace.MyClass.MyTest", result.FullyQualifiedName); + } + + [Fact] + public void TestResult_FullyQualifiedName_FallsBackToTestCaseTitle() + { + // Arrange + var result = new TestResult + { + Id = 1, + AutomatedTestName = null, + TestCaseTitle = "My Test Title", + }; + + // Assert + Assert.Equal("My Test Title", result.FullyQualifiedName); + } + + [Fact] + public void TestResult_FullyQualifiedName_FallsBackToId() + { + // Arrange + var result = new TestResult + { + Id = 42, + AutomatedTestName = null, + TestCaseTitle = null, + }; + + // Assert + Assert.Equal("Unknown-42", result.FullyQualifiedName); + } + + [Fact] + public void Build_DefaultValues() + { + // Arrange + var build = new Build(); + + // Assert + Assert.Equal(0, build.Id); + Assert.Null(build.BuildNumber); + Assert.Equal(BuildStatus.None, build.Status); + Assert.Null(build.Result); + Assert.Null(build.SourceVersion); + } + + [Fact] + public void TestFailureDiff_RequiredProperties() + { + // Arrange & Act + var diff = new TestFailureDiff + { + NewFailures = new List { "Test1", "Test2" }, + ConsistentFailures = new List(), + }; + + // Assert + Assert.Equal(2, diff.NewFailures.Count); + Assert.Empty(diff.ConsistentFailures); + } +} diff --git a/src/tools/auto-bisect/tests/TestDifferTests.cs b/src/tools/auto-bisect/tests/TestDifferTests.cs new file mode 100644 index 00000000000000..5061df79d33ada --- /dev/null +++ b/src/tools/auto-bisect/tests/TestDifferTests.cs @@ -0,0 +1,194 @@ +using System.Collections.Generic; +using AutoBisect; +using Xunit; + +namespace AutoBisect.Tests; + +public class TestDifferTests +{ + [Fact] + public void ComputeDiff_IdenticalFailures_AllConsistent() + { + // Arrange - only failed tests are passed to the differ + var goodFailures = new List + { + new() { AutomatedTestName = "Test2", Outcome = TestOutcome.Failed }, + }; + + var badFailures = new List + { + new() { AutomatedTestName = "Test2", Outcome = TestOutcome.Failed }, + }; + + // Act + var diff = TestDiffer.ComputeDiff(goodFailures, badFailures); + + // Assert + Assert.Empty(diff.NewFailures); + Assert.Single(diff.ConsistentFailures); + Assert.Equal("Test2", diff.ConsistentFailures[0]); + } + + [Fact] + public void ComputeDiff_NewFailure_DetectedCorrectly() + { + // Arrange - good build has no failures, bad build has one + var goodFailures = new List(); + + var badFailures = new List + { + new() { AutomatedTestName = "Test2", Outcome = TestOutcome.Failed }, + }; + + // Act + var diff = TestDiffer.ComputeDiff(goodFailures, badFailures); + + // Assert + Assert.Single(diff.NewFailures); + Assert.Equal("Test2", diff.NewFailures[0]); + Assert.Empty(diff.ConsistentFailures); + } + + [Fact] + public void ComputeDiff_FailureOnlyInGood_NotReported() + { + // Arrange - good build has a failure that bad build doesn't have + // (test was fixed, but we can't report "fixed" without passed test data) + var goodFailures = new List + { + new() { AutomatedTestName = "Test1", Outcome = TestOutcome.Failed }, + }; + + var badFailures = new List(); + + // Act + var diff = TestDiffer.ComputeDiff(goodFailures, badFailures); + + // Assert + Assert.Empty(diff.NewFailures); + Assert.Empty(diff.ConsistentFailures); + } + + [Fact] + public void ComputeDiff_MultipleChanges_CategorizedCorrectly() + { + // Arrange - only failures are passed + var goodFailures = new List + { + new() { AutomatedTestName = "AlwaysFails", Outcome = TestOutcome.Failed }, + new() { AutomatedTestName = "GetsFixed", Outcome = TestOutcome.Failed }, + }; + + var badFailures = new List + { + new() { AutomatedTestName = "BecomesFailing", Outcome = TestOutcome.Failed }, + new() { AutomatedTestName = "AlwaysFails", Outcome = TestOutcome.Failed }, + }; + + // Act + var diff = TestDiffer.ComputeDiff(goodFailures, badFailures); + + // Assert + Assert.Single(diff.NewFailures); + Assert.Contains("BecomesFailing", diff.NewFailures); + + Assert.Single(diff.ConsistentFailures); + Assert.Contains("AlwaysFails", diff.ConsistentFailures); + } + + [Fact] + public void ComputeDiff_CaseInsensitive_TreatsAsSameTest() + { + // Arrange + var goodFailures = new List(); + + var badFailures = new List + { + new() { AutomatedTestName = "MYNAMESPACE.MYTEST", Outcome = TestOutcome.Failed }, + }; + + // Act + var diff = TestDiffer.ComputeDiff(goodFailures, badFailures); + + // Assert + Assert.Single(diff.NewFailures); + } + + [Fact] + public void ComputeDiff_CaseInsensitive_ConsistentFailure() + { + // Arrange + var goodFailures = new List + { + new() { AutomatedTestName = "MyNamespace.MyTest", Outcome = TestOutcome.Failed }, + }; + + var badFailures = new List + { + new() { AutomatedTestName = "MYNAMESPACE.MYTEST", Outcome = TestOutcome.Failed }, + }; + + // Act + var diff = TestDiffer.ComputeDiff(goodFailures, badFailures); + + // Assert + Assert.Empty(diff.NewFailures); + Assert.Single(diff.ConsistentFailures); + } + + [Fact] + public void ComputeDiff_EmptyResults_ReturnsEmptyDiff() + { + // Arrange + var goodFailures = new List(); + var badFailures = new List(); + + // Act + var diff = TestDiffer.ComputeDiff(goodFailures, badFailures); + + // Assert + Assert.Empty(diff.NewFailures); + Assert.Empty(diff.ConsistentFailures); + } + + [Fact] + public void ComputeDiff_OnlyGoodHasFailures_NoNewFailures() + { + // Arrange + var goodFailures = new List + { + new() { AutomatedTestName = "Test2", Outcome = TestOutcome.Failed }, + }; + var badFailures = new List(); + + // Act + var diff = TestDiffer.ComputeDiff(goodFailures, badFailures); + + // Assert + Assert.Empty(diff.NewFailures); + Assert.Empty(diff.ConsistentFailures); + } + + [Fact] + public void ComputeDiff_ResultsAreSorted() + { + // Arrange + var goodFailures = new List(); + + var badFailures = new List + { + new() { AutomatedTestName = "Zebra", Outcome = TestOutcome.Failed }, + new() { AutomatedTestName = "Apple", Outcome = TestOutcome.Failed }, + new() { AutomatedTestName = "Mango", Outcome = TestOutcome.Failed }, + }; + + // Act + var diff = TestDiffer.ComputeDiff(goodFailures, badFailures); + + // Assert + Assert.Equal(3, diff.NewFailures.Count); + Assert.Equal("Apple", diff.NewFailures[0]); + Assert.Equal("Mango", diff.NewFailures[1]); + Assert.Equal("Zebra", diff.NewFailures[2]); + } +}