From 3ac172a79015ac7d2728e20439a94df2dd2ac95e Mon Sep 17 00:00:00 2001 From: Jose Perez Rodriguez Date: Fri, 9 May 2025 18:21:55 +0000 Subject: [PATCH 01/26] Merged PR 49897: Changes in preparation of 9.5 stable release Changes in preparation of 9.5 stable release ---- #### AI description (iteration 1) #### PR Classification This PR prepares the upcoming 9.5 stable release by updating dependency versions and adjusting build and pipeline configurations. #### PR Summary The changes update version numbers and release settings while refining the CI pipeline and NuGet feed setup for a stable release build. - **`eng/Version.Details.xml` & `eng/Versions.props`**: Updated dependency versions from 9.0.4 to 9.0.5 (and LTS versions from 8.0.15 to 8.0.16) and enabled release-specific properties. - **`NuGet.config`**: Revised package source mappings by adding and disabling internal feed sources. - **`azure-pipelines.yml`**: Removed the code coverage stage and updated stage dependencies. - **`eng/pipelines/templates/BuildAndTest.yml`**: Added tasks for setting up private NuGet feed credentials on both Windows (PowerShell) and non-Windows (Bash) agents. - **`Directory.Build.props`**: Disabled the NU1507 warning to accommodate the internal feed configuration changes. --- Directory.Build.props | 5 + NuGet.config | 98 +++++++++--- azure-pipelines.yml | 46 ------ eng/Version.Details.xml | 188 +++++++++++------------ eng/Versions.props | 122 +++++++-------- eng/pipelines/templates/BuildAndTest.yml | 32 +++- 6 files changed, 262 insertions(+), 229 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 0af806af628..0c0fcf22bfd 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -34,6 +34,11 @@ $(NetCoreTargetFrameworks) + + + $(NoWarn);NU1507 + + false latest diff --git a/NuGet.config b/NuGet.config index 0fedd015e82..0d98374a4f3 100644 --- a/NuGet.config +++ b/NuGet.config @@ -4,10 +4,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -18,35 +56,51 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3ec5e3d1cdb..0052dc9f706 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -239,51 +239,6 @@ extends: isWindows: false warnAsError: 0 - # ---------------------------------------------------------------- - # This stage performs quality gates enforcements - # ---------------------------------------------------------------- - - stage: codecoverage - displayName: CodeCoverage - dependsOn: - - build - condition: and(succeeded('build'), ne(variables['SkipQualityGates'], 'true')) - variables: - - template: /eng/common/templates-official/variables/pool-providers.yml@self - jobs: - - template: /eng/common/templates-official/jobs/jobs.yml@self - parameters: - enableMicrobuild: true - enableTelemetry: true - runAsPublic: ${{ variables['runAsPublic'] }} - workspace: - clean: all - - # ---------------------------------------------------------------- - # This stage downloads the code coverage reports from the build jobs, - # merges those and validates the combined test coverage. - # ---------------------------------------------------------------- - jobs: - - job: CodeCoverageReport - timeoutInMinutes: 180 - - pool: - name: NetCore1ESPool-Internal - image: 1es-mariner-2 - os: linux - - preSteps: - - checkout: self - clean: true - persistCredentials: true - fetchDepth: 1 - - steps: - - script: $(Build.SourcesDirectory)/build.sh --ci --restore - displayName: Init toolset - - - template: /eng/pipelines/templates/VerifyCoverageReport.yml - - # ---------------------------------------------------------------- # This stage only performs a build treating warnings as errors # to detect any kind of code style violations @@ -339,7 +294,6 @@ extends: parameters: validateDependsOn: - build - - codecoverage - correctness publishingInfraVersion: 3 enableSymbolValidation: false diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 79b661b0db6..859de22a83f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,196 +1,196 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-efcore - 55700ce7d51b40ea546f817fd4947a6bae50be07 + 6765359588e8b38bab2a7974db9398432703828f diff --git a/eng/Versions.props b/eng/Versions.props index 96ec3bac1db..b2dd32f628d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,14 +11,14 @@ - false + true - + release true @@ -34,55 +34,55 @@ --> - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 - 9.0.4 + 9.0.5 9.0.0-beta.25225.6 @@ -108,8 +108,8 @@ 8.0.1 8.0.0 8.0.2 - 8.0.14 - 8.0.14 + 8.0.16 + 8.0.16 8.0.0 8.0.1 8.0.1 @@ -126,15 +126,17 @@ 8.0.5 8.0.0 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + + 8.0.16 - $(MinimumSupportedTfmForPackaging) + $(NetCoreTargetFrameworks) Microsoft.Extensions.AI.Evaluation.Console $(NoWarn);EA0000 @@ -22,6 +20,15 @@ 0 + + + false + + From 49bb2c5ea0326f7f055ac42c538e59834c159b13 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 9 May 2025 10:04:58 -0400 Subject: [PATCH 11/26] Add BinaryEmbedding (#6398) * Add BinaryEmbedding Also: - Renames the polymorphic discriminators to conform with typical lingo for these types. - Adds an Embedding.Dimensions virtual property. --- .../Embeddings/BinaryEmbedding.cs | 111 ++++++++++++++++++ .../Embeddings/Embedding.cs | 20 +++- .../Embeddings/Embedding{T}.cs | 5 + .../Embeddings/BinaryEmbeddingTests.cs | 95 +++++++++++++++ .../Embeddings/EmbeddingTests.cs | 34 +++++- .../BinaryEmbedding.cs | 16 --- .../EmbeddingGeneratorIntegrationTests.cs | 12 +- .../QuantizationEmbeddingGenerator.cs | 5 +- 8 files changed, 270 insertions(+), 28 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/BinaryEmbedding.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs delete mode 100644 test/Libraries/Microsoft.Extensions.AI.Integration.Tests/BinaryEmbedding.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/BinaryEmbedding.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/BinaryEmbedding.cs new file mode 100644 index 00000000000..2261fd97949 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/BinaryEmbedding.cs @@ -0,0 +1,111 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Buffers; +using System.Collections; +using System.ComponentModel; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents an embedding composed of a bit vector. +public sealed class BinaryEmbedding : Embedding +{ + /// The embedding vector this embedding represents. + private BitArray _vector; + + /// Initializes a new instance of the class with the embedding vector. + /// The embedding vector this embedding represents. + /// is . + public BinaryEmbedding(BitArray vector) + { + _vector = Throw.IfNull(vector); + } + + /// Gets or sets the embedding vector this embedding represents. + [JsonConverter(typeof(VectorConverter))] + public BitArray Vector + { + get => _vector; + set => _vector = Throw.IfNull(value); + } + + /// + [JsonIgnore] + public override int Dimensions => _vector.Length; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class VectorConverter : JsonConverter + { + /// + public override BitArray Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + _ = Throw.IfNull(typeToConvert); + _ = Throw.IfNull(options); + + if (reader.TokenType != JsonTokenType.String) + { + throw new JsonException("Expected string property."); + } + + ReadOnlySpan utf8; + byte[]? tmpArray = null; + if (!reader.HasValueSequence && !reader.ValueIsEscaped) + { + utf8 = reader.ValueSpan; + } + else + { + // This path should be rare. + int length = reader.HasValueSequence ? checked((int)reader.ValueSequence.Length) : reader.ValueSpan.Length; + tmpArray = ArrayPool.Shared.Rent(length); + utf8 = tmpArray.AsSpan(0, reader.CopyString(tmpArray)); + } + + BitArray result = new(utf8.Length); + + for (int i = 0; i < utf8.Length; i++) + { + result[i] = utf8[i] switch + { + (byte)'0' => false, + (byte)'1' => true, + _ => throw new JsonException("Expected binary character sequence.") + }; + } + + if (tmpArray is not null) + { + ArrayPool.Shared.Return(tmpArray); + } + + return result; + } + + /// + public override void Write(Utf8JsonWriter writer, BitArray value, JsonSerializerOptions options) + { + _ = Throw.IfNull(writer); + _ = Throw.IfNull(value); + _ = Throw.IfNull(options); + + int length = value.Length; + + byte[] tmpArray = ArrayPool.Shared.Rent(length); + + Span utf8 = tmpArray.AsSpan(0, length); + for (int i = 0; i < utf8.Length; i++) + { + utf8[i] = value[i] ? (byte)'1' : (byte)'0'; + } + + writer.WriteStringValue(utf8); + + ArrayPool.Shared.Return(tmpArray); + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs index 19b8feaa182..d6596e1e53e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Diagnostics; using System.Text.Json.Serialization; namespace Microsoft.Extensions.AI; @@ -9,13 +10,15 @@ namespace Microsoft.Extensions.AI; /// Represents an embedding generated by a . /// This base class provides metadata about the embedding. Derived types provide the concrete data contained in the embedding. [JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] +[JsonDerivedType(typeof(BinaryEmbedding), typeDiscriminator: "binary")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "uint8")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "int8")] #if NET -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "halves")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "float16")] #endif -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "floats")] -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "doubles")] -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "bytes")] -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "sbytes")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "float32")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "float64")] +[DebuggerDisplay("Dimensions = {Dimensions}")] public class Embedding { /// Initializes a new instance of the class. @@ -26,6 +29,13 @@ protected Embedding() /// Gets or sets a timestamp at which the embedding was created. public DateTimeOffset? CreatedAt { get; set; } + /// Gets the dimensionality of the embedding vector. + /// + /// This value corresponds to the number of elements in the embedding vector. + /// + [JsonIgnore] + public virtual int Dimensions { get; } + /// Gets or sets the model ID using in the creation of the embedding. public string? ModelId { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs index c80e20dfda4..22bc02f2f3f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Text.Json.Serialization; namespace Microsoft.Extensions.AI; @@ -19,4 +20,8 @@ public Embedding(ReadOnlyMemory vector) /// Gets or sets the embedding vector this embedding represents. public ReadOnlyMemory Vector { get; set; } + + /// + [JsonIgnore] + public override int Dimensions => Vector.Length; } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs new file mode 100644 index 00000000000..c75d715466e --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections; +using System.Linq; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class BinaryEmbeddingTests +{ + [Fact] + public void Ctor_Roundtrips() + { + BitArray vector = new BitArray(new bool[] { false, true, false, true }); + + BinaryEmbedding e = new(vector); + Assert.Same(vector, e.Vector); + Assert.Null(e.ModelId); + Assert.Null(e.CreatedAt); + Assert.Null(e.AdditionalProperties); + } + + [Fact] + public void Properties_Roundtrips() + { + BitArray vector = new BitArray(new bool[] { false, true, false, true }); + + BinaryEmbedding e = new(vector); + + Assert.Same(vector, e.Vector); + BitArray newVector = new BitArray(new bool[] { true, false, true, false }); + e.Vector = newVector; + Assert.Same(newVector, e.Vector); + + Assert.Null(e.ModelId); + e.ModelId = "text-embedding-3-small"; + Assert.Equal("text-embedding-3-small", e.ModelId); + + Assert.Null(e.CreatedAt); + DateTimeOffset createdAt = DateTimeOffset.Parse("2022-01-01T00:00:00Z"); + e.CreatedAt = createdAt; + Assert.Equal(createdAt, e.CreatedAt); + + Assert.Null(e.AdditionalProperties); + AdditionalPropertiesDictionary props = new(); + e.AdditionalProperties = props; + Assert.Same(props, e.AdditionalProperties); + } + + [Fact] + public void Serialization_Roundtrips() + { + foreach (int length in Enumerable.Range(0, 64).Concat(new[] { 10_000 })) + { + bool[] bools = new bool[length]; + Random r = new(42); + for (int i = 0; i < length; i++) + { + bools[i] = r.Next(2) != 0; + } + + BitArray vector = new BitArray(bools); + BinaryEmbedding e = new(vector); + + string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); + Assert.Equal($$"""{"$type":"binary","vector":"{{string.Concat(vector.Cast().Select(b => b ? '1' : '0'))}}"}""", json); + + BinaryEmbedding result = Assert.IsType(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + Assert.Equal(e.Vector, result.Vector); + } + } + + [Fact] + public void Derialization_SupportsEncodedBits() + { + BinaryEmbedding result = Assert.IsType(JsonSerializer.Deserialize( + """{"$type":"binary","vector":"\u0030\u0031\u0030\u0031\u0030\u0031"}""", + TestJsonSerializerContext.Default.Embedding)); + + Assert.Equal(new BitArray(new[] { false, true, false, true, false, true }), result.Vector); + } + + [Theory] + [InlineData("""{"$type":"binary","vector":"\u0030\u0032"}""")] + [InlineData("""{"$type":"binary","vector":"02"}""")] + [InlineData("""{"$type":"binary","vector":" "}""")] + [InlineData("""{"$type":"binary","vector":10101}""")] + public void Derialization_InvalidBinaryEmbedding_Throws(string json) + { + Assert.Throws(() => JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs index 45fcce8ba63..c3809782006 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs @@ -14,7 +14,7 @@ public class EmbeddingTests public void Embedding_Ctor_Roundtrips() { float[] floats = [1f, 2f, 3f]; - UsageDetails usage = new(); + AdditionalPropertiesDictionary props = []; var createdAt = DateTimeOffset.Parse("2022-01-01T00:00:00Z"); const string Model = "text-embedding-3-small"; @@ -35,6 +35,32 @@ public void Embedding_Ctor_Roundtrips() Assert.Same(floats, array.Array); } + [Fact] + public void Embedding_Byte_SerializationRoundtrips() + { + byte[] bytes = [1, 2, 3]; + Embedding e = new(bytes); + + string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); + Assert.Equal("""{"$type":"uint8","vector":"AQID"}""", json); + + Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); + } + + [Fact] + public void Embedding_SByte_SerializationRoundtrips() + { + sbyte[] bytes = [1, 2, 3]; + Embedding e = new(bytes); + + string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); + Assert.Equal("""{"$type":"int8","vector":[1,2,3]}""", json); + + Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); + } + #if NET [Fact] public void Embedding_Half_SerializationRoundtrips() @@ -43,7 +69,7 @@ public void Embedding_Half_SerializationRoundtrips() Embedding e = new(halfs); string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); - Assert.Equal("""{"$type":"halves","vector":[1,2,3]}""", json); + Assert.Equal("""{"$type":"float16","vector":[1,2,3]}""", json); Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); @@ -57,7 +83,7 @@ public void Embedding_Single_SerializationRoundtrips() Embedding e = new(floats); string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); - Assert.Equal("""{"$type":"floats","vector":[1,2,3]}""", json); + Assert.Equal("""{"$type":"float32","vector":[1,2,3]}""", json); Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); @@ -70,7 +96,7 @@ public void Embedding_Double_SerializationRoundtrips() Embedding e = new(floats); string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); - Assert.Equal("""{"$type":"doubles","vector":[1,2,3]}""", json); + Assert.Equal("""{"$type":"float64","vector":[1,2,3]}""", json); Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/BinaryEmbedding.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/BinaryEmbedding.cs deleted file mode 100644 index f538d1476b0..00000000000 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/BinaryEmbedding.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; - -namespace Microsoft.Extensions.AI; - -internal sealed class BinaryEmbedding : Embedding -{ - public BinaryEmbedding(ReadOnlyMemory bits) - { - Bits = bits; - } - - public ReadOnlyMemory Bits { get; } -} diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs index 1188e899e4d..1504d0d2488 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs @@ -2,6 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +#if NET +using System.Collections; +#endif using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; @@ -148,7 +151,14 @@ public async Task Quantization_Binary_EmbeddingsCompareSuccessfully() { for (int j = 0; j < embeddings.Count; j++) { - distances[i, j] = TensorPrimitives.HammingBitDistance(embeddings[i].Bits.Span, embeddings[j].Bits.Span); + distances[i, j] = TensorPrimitives.HammingBitDistance(ToArray(embeddings[i].Vector), ToArray(embeddings[j].Vector)); + + static byte[] ToArray(BitArray array) + { + byte[] result = new byte[(array.Length + 7) / 8]; + array.CopyTo(result, 0); + return result; + } } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs index 3bf33988146..ea87408da38 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections; using System.Collections.Generic; using System.Linq; #if NET @@ -46,12 +47,12 @@ private static BinaryEmbedding QuantizeToBinary(Embedding embedding) { ReadOnlySpan vector = embedding.Vector.Span; - var result = new byte[(int)Math.Ceiling(vector.Length / 8.0)]; + var result = new BitArray(vector.Length); for (int i = 0; i < vector.Length; i++) { if (vector[i] > 0) { - result[i / 8] |= (byte)(1 << (i % 8)); + result[i / 8] = true; } } From 6828de68d4b5ae7eb09492b0e77d17641d7927fc Mon Sep 17 00:00:00 2001 From: Peter Waldschmidt Date: Thu, 8 May 2025 15:53:11 -0400 Subject: [PATCH 12/26] Add some additional documentation around usage of cache, and CSP properties on report (#6377) * Add documentation around proper usage of IDistributedCache * Add Content-Security-Policy to prevent page from calling into other sites. * Remove remark about IDistributedCache usage * Fix package-lock.json * Remove start tag. --- .../TypeScript/html-report/index.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html index 8169711aca6..c34388e543c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html @@ -8,8 +8,7 @@ - + Codestin Search App From 118d4e68e6d959039c35a2a33511bbfc9fae7cc5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 19:02:39 -0700 Subject: [PATCH 13/26] Bump vite from 6.2.6 to 6.3.4 in /src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript (#6354) * Bump vite Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 6.2.6 to 6.3.4. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/main/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v6.3.4/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-version: 6.3.4 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shyam Namboodiripad --- .../TypeScript/package-lock.json | 87 +++++++++++++++++-- .../TypeScript/package.json | 2 +- 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json index 3e77d76226f..6e76e88aa03 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json @@ -33,7 +33,7 @@ "tfx-cli": "^0.21.0", "typescript": "^5.5.3", "typescript-eslint": "^8.27.0", - "vite": "^6.2.6", + "vite": "^6.3.4", "vite-plugin-singlefile": "^2.0.2" } }, @@ -8274,6 +8274,51 @@ "node": "*" } }, + "node_modules/tinyglobby": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.13.tgz", + "integrity": "sha512-mEwzpUgrLySlveBwEVDMKk5B57bhLPYovRfPAXD5gA/98Opn0rCDj3GtLwFvCvH5RK9uPCExUROW5NjDwvqkxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.4.4", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.4.4", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", + "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/tinytim": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/tinytim/-/tinytim-0.1.1.tgz", @@ -8764,14 +8809,18 @@ } }, "node_modules/vite": { - "version": "6.2.6", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.6.tgz", - "integrity": "sha512-9xpjNl3kR4rVDZgPNdTL0/c6ao4km69a/2ihNQbcANz8RuCOK3hQBmLSJf3bRKVQjVMda+YvizNE8AwvogcPbw==", + "version": "6.3.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.3.5.tgz", + "integrity": "sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==", "dev": true, + "license": "MIT", "dependencies": { "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", "postcss": "^8.5.3", - "rollup": "^4.30.1" + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" }, "bin": { "vite": "bin/vite.js" @@ -8850,6 +8899,34 @@ "vite": "^5.4.11 || ^6.0.0" } }, + "node_modules/vite/node_modules/fdir": { + "version": "6.4.4", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", + "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/walkdir": { "version": "0.0.11", "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.0.11.tgz", diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json index c2505b5d87a..0e32f4ae6f7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json @@ -34,7 +34,7 @@ "tfx-cli": "^0.21.0", "typescript": "^5.5.3", "typescript-eslint": "^8.27.0", - "vite": "^6.2.6", + "vite": "^6.3.4", "vite-plugin-singlefile": "^2.0.2" } } From 6753e516632cfac448032d6e781e70d40c7f42e6 Mon Sep 17 00:00:00 2001 From: Shyam N Date: Fri, 9 May 2025 11:33:25 -0700 Subject: [PATCH 14/26] Some API related fixes for the evaluation libraries (#6402) * Rename IResultStore and IResponseCacheProvider IResultStore -> IEvaluationResultStore and IResponseCacheProvider -> IEvaluationResponseCacheProvider * Include missing EvaluationContextConverter in AzureStorageJsonUtilities Also use linked files to avoid the need to duplicate code. * Reorder enum members The new order goes from least desirable rating to most desirable. * Refactor extension method overloads Implement overloads that take ChatMessage by calling corresponding overloads that take ChatResponse. * Refactor AddTurnDetails to support adding details for multiple turns Adding single turns continues to be supported via a params array overload. * Add missing parameter for timeToLiveForCacheEntries to DiskBasedReportingConfiguration This was missed in an earlier PR that introduced the timeToLiveForCacheEntries on the constructor of DiskBasedResponseCacheProvider. Also reorder constructor parameters for AzureStorageReportingConfiguration so that the parameters for caching apear next to each other and so that the parameter ordering is aligned with DiskBasedReportingConfiguration. * Minor formatting changes --- .../Commands/CleanCacheCommand.cs | 2 +- .../Commands/CleanResultsCommand.cs | 2 +- .../Commands/ReportCommand.cs | 2 +- .../AzureStorageCamelCaseEnumConverter.cs | 11 -------- .../AzureStorageJsonUtilities.cs | 12 ++++---- .../AzureStorageTimeSpanConverter.cs | 17 ----------- ...sions.AI.Evaluation.Reporting.Azure.csproj | 6 ++++ .../AzureStorageReportingConfiguration.cs | 14 +++++----- .../AzureStorageResponseCacheProvider.cs | 6 ++-- .../Storage/AzureStorageResultStore.cs | 6 ++-- .../CSharp/ChatDetailsExtensions.cs | 28 +++++++++++++++---- .../CSharp/Defaults.cs | 2 +- ...cs => IEvaluationResponseCacheProvider.cs} | 26 +++++++++-------- ...sultStore.cs => IEvaluationResultStore.cs} | 2 +- .../CSharp/JsonSerialization/JsonUtilities.cs | 5 ++-- .../CSharp/ReportingConfiguration.cs | 19 +++++++------ .../CSharp/ScenarioRun.cs | 6 ++-- .../CSharp/ScenarioRunExtensions.cs | 19 ++++--------- .../DiskBasedReportingConfiguration.cs | 11 ++++++-- .../Storage/DiskBasedResponseCacheProvider.cs | 14 ++++++---- .../CSharp/Storage/DiskBasedResultStore.cs | 4 +-- .../AIContentExtensions.cs | 1 + .../ContentSafetyServicePayloadFormat.cs | 2 +- .../EvaluationRating.cs | 16 +++++------ .../EvaluatorExtensions.cs | 19 ++++--------- .../QualityEvaluatorTests.cs | 2 +- .../AzureStorage/AzureResponseCacheTests.cs | 4 +-- .../AzureStorage/AzureResultStoreTests.cs | 2 +- .../DiskBased/DiskBasedResponseCacheTests.cs | 4 +-- .../DiskBased/DiskBasedResultStoreTests.cs | 2 +- .../ResponseCacheTester.cs | 16 +++++------ .../ResultStoreTester.cs | 17 +++++------ 32 files changed, 147 insertions(+), 152 deletions(-) delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs rename src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/{IResponseCacheProvider.cs => IEvaluationResponseCacheProvider.cs} (58%) rename src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/{IResultStore.cs => IEvaluationResultStore.cs} (99%) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs index 08f035d55eb..b0d975edb43 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs @@ -18,7 +18,7 @@ internal sealed class CleanCacheCommand(ILogger logger) { internal async Task InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default) { - IResponseCacheProvider cacheProvider; + IEvaluationResponseCacheProvider cacheProvider; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs index 59635dc0530..8d6617d8302 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs @@ -23,7 +23,7 @@ internal async Task InvokeAsync( int lastN, CancellationToken cancellationToken = default) { - IResultStore resultStore; + IEvaluationResultStore resultStore; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs index f2466923bd8..2611695e542 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs @@ -28,7 +28,7 @@ internal async Task InvokeAsync( Format format, CancellationToken cancellationToken = default) { - IResultStore resultStore; + IEvaluationResultStore resultStore; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs deleted file mode 100644 index 2ec6cdb801f..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; - -internal sealed class AzureStorageCamelCaseEnumConverter() : - JsonStringEnumConverter(JsonNamingPolicy.CamelCase) - where TEnum : struct, System.Enum; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs index 23b2ae9c88c..b36c8d8bd56 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -11,7 +12,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; internal static partial class AzureStorageJsonUtilities { - [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] + [SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] internal static class Default { private static JsonSerializerOptions? _options; @@ -24,6 +25,7 @@ internal static class Compact { private static JsonSerializerOptions? _options; internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: false); + internal static JsonTypeInfo CacheEntryTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo ScenarioRunResultTypeInfo => Options.GetTypeInfo(); } @@ -45,14 +47,14 @@ private static JsonSerializerOptions CreateJsonSerializerOptions(bool writeInden [JsonSerializable(typeof(CacheEntry))] [JsonSourceGenerationOptions( Converters = [ - typeof(AzureStorageCamelCaseEnumConverter), - typeof(AzureStorageCamelCaseEnumConverter), - typeof(AzureStorageTimeSpanConverter) + typeof(CamelCaseEnumConverter), + typeof(CamelCaseEnumConverter), + typeof(TimeSpanConverter), + typeof(EvaluationContextConverter) ], WriteIndented = true, IgnoreReadOnlyProperties = false, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] private sealed partial class JsonContext : JsonSerializerContext; - } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs deleted file mode 100644 index 0c064ededd3..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; - -internal sealed class AzureStorageTimeSpanConverter : JsonConverter -{ - public override TimeSpan Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - => TimeSpan.FromSeconds(reader.GetDouble()); - - public override void Write(Utf8JsonWriter writer, TimeSpan value, JsonSerializerOptions options) - => writer.WriteNumberValue(value.TotalSeconds); -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj index 669bbab7556..237df014d0d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj @@ -17,6 +17,12 @@ 0 + + + + + + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs index 9302107b926..fafd8639b34 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs @@ -24,10 +24,6 @@ public static class AzureStorageReportingConfiguration /// /// The set of s that should be invoked to evaluate AI responses. /// - /// - /// An optional that specifies the maximum amount of time that cached AI responses should - /// survive in the cache before they are considered expired and evicted. - /// /// /// A that specifies the that is used by AI-based /// included in the returned . Can be omitted if @@ -36,6 +32,10 @@ public static class AzureStorageReportingConfiguration /// /// to enable caching of AI responses; otherwise. /// + /// + /// An optional that specifies the maximum amount of time that cached AI responses should + /// survive in the cache before they are considered expired and evicted. + /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI /// responses. See for more information about this concept. @@ -63,21 +63,21 @@ public static class AzureStorageReportingConfiguration public static ReportingConfiguration Create( DataLakeDirectoryClient client, IEnumerable evaluators, - TimeSpan? timeToLiveForCacheEntries = null, ChatConfiguration? chatConfiguration = null, bool enableResponseCaching = true, + TimeSpan? timeToLiveForCacheEntries = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, IEnumerable? tags = null) #pragma warning restore S107 { - IResponseCacheProvider? responseCacheProvider = + IEvaluationResponseCacheProvider? responseCacheProvider = chatConfiguration is not null && enableResponseCaching ? new AzureStorageResponseCacheProvider(client, timeToLiveForCacheEntries) : null; - IResultStore resultStore = new AzureStorageResultStore(client); + IEvaluationResultStore resultStore = new AzureStorageResultStore(client); return new ReportingConfiguration( evaluators, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs index a890fa80332..6c6d1431a1a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs @@ -15,8 +15,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An that returns an that can cache AI responses -/// for a particular under an Azure Storage container. +/// An that returns an that can cache AI +/// responses for a particular under an Azure Storage container. /// /// /// A with access to an Azure Storage container under which the cached AI @@ -28,7 +28,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// public sealed class AzureStorageResponseCacheProvider( DataLakeDirectoryClient client, - TimeSpan? timeToLiveForCacheEntries = null) : IResponseCacheProvider + TimeSpan? timeToLiveForCacheEntries = null) : IEvaluationResponseCacheProvider { private readonly Func _provideDateTime = () => DateTime.Now; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs index 70d988abe74..71682f13651 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs @@ -20,14 +20,14 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An implementation that stores s under an Azure Storage -/// container. +/// An implementation that stores s under an Azure +/// Storage container. /// /// /// A with access to an Azure Storage container under which the /// s should be stored. /// -public sealed class AzureStorageResultStore(DataLakeDirectoryClient client) : IResultStore +public sealed class AzureStorageResultStore(DataLakeDirectoryClient client) : IEvaluationResultStore { private const string ResultsRootPrefix = "results"; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs index 006dfc741e8..71afe53217a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI.Evaluation.Reporting; @@ -11,19 +12,36 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; public static class ChatDetailsExtensions { /// - /// Adds for a particular LLM chat conversation turn to the + /// Adds for one or more LLM chat conversation turns to the /// collection. /// /// - /// The object to which the is to be added. + /// The object to which the are to be added. /// /// - /// The for a particular LLM chat conversation turn. + /// The for one or more LLM chat conversation turns. /// - public static void AddTurnDetails(this ChatDetails chatDetails, ChatTurnDetails turnDetails) + public static void AddTurnDetails(this ChatDetails chatDetails, IEnumerable turnDetails) { _ = Throw.IfNull(chatDetails); + _ = Throw.IfNull(turnDetails); - chatDetails.TurnDetails.Add(turnDetails); + foreach (ChatTurnDetails t in turnDetails) + { + chatDetails.TurnDetails.Add(t); + } } + + /// + /// Adds for one or more LLM chat conversation turns to the + /// collection. + /// + /// + /// The object to which the are to be added. + /// + /// + /// The for one or more LLM chat conversation turns. + /// + public static void AddTurnDetails(this ChatDetails chatDetails, params ChatTurnDetails[] turnDetails) + => chatDetails.AddTurnDetails(turnDetails as IEnumerable); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs index 095bdee575c..f25fa074430 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs @@ -27,7 +27,7 @@ public static class Defaults /// /// Gets a that specifies the default amount of time that cached AI responses should survive - /// in the 's cache before they are considered expired and evicted. + /// in the 's cache before they are considered expired and evicted. /// public static TimeSpan DefaultTimeToLiveForCacheEntries { get; } = TimeSpan.FromDays(14); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs similarity index 58% rename from src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs rename to src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs index 6bc8ce25432..1859124a98f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs @@ -12,26 +12,28 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; /// . /// /// -/// can be used to set up caching of AI-generated responses (both the AI responses -/// under evaluation as well as the AI responses for the evaluations themselves). When caching is enabled, the AI -/// responses associated with each are stored in the that is -/// returned from this . So long as the inputs (such as the content included in the -/// requests, the AI model being invoked etc.) remain unchanged, subsequent evaluations of the same -/// use the cached responses instead of invoking the AI model to generate new ones. Bypassing -/// the AI model when the inputs remain unchanged results in faster execution at a lower cost. +/// can be used to set up caching of AI-generated responses (both the AI +/// responses under evaluation as well as the AI responses for the evaluations themselves). When caching is enabled, +/// the AI responses associated with each are stored in the +/// that is returned from this . So long as the inputs (such as the +/// content included in the requests, the AI model being invoked etc.) remain unchanged, subsequent evaluations of the +/// same use the cached responses instead of invoking the AI model to generate new ones. +/// Bypassing the AI model when the inputs remain unchanged results in faster execution at a lower cost. /// -public interface IResponseCacheProvider +public interface IEvaluationResponseCacheProvider { /// - /// Returns an that caches the AI responses associated with a particular - /// . + /// Returns an that caches all the AI responses associated with the + /// with the supplied and + /// . /// /// The . /// The . /// A that can cancel the operation. /// - /// An that caches the AI responses associated with a particular - /// . + /// An that caches all the AI responses associated with the + /// with the supplied and + /// . /// ValueTask GetCacheAsync( string scenarioName, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs similarity index 99% rename from src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs rename to src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs index 3f3dea6cc7a..202a6305cd3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs @@ -10,7 +10,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; /// /// Represents a store for s. /// -public interface IResultStore +public interface IEvaluationResultStore { /// /// Returns s for s filtered by the specified diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs index 9ba74009433..3a8c2af1ce2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -12,7 +13,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; internal static partial class JsonUtilities { - [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] + [SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] internal static class Default { private static JsonSerializerOptions? _options; @@ -45,7 +46,7 @@ private static JsonSerializerOptions CreateJsonSerializerOptions(bool writeInden return options; } - [JsonSerializable(typeof(EvaluationResult))] + [JsonSerializable(typeof(ScenarioRunResult))] [JsonSerializable(typeof(Dataset))] [JsonSerializable(typeof(CacheEntry))] [JsonSourceGenerationOptions( diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs index 68a73338d88..130586de930 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs @@ -27,9 +27,10 @@ public sealed class ReportingConfiguration public IReadOnlyList Evaluators { get; } /// - /// Gets the that should be used to persist the s. + /// Gets the that should be used to persist the + /// s. /// - public IResultStore ResultStore { get; } + public IEvaluationResultStore ResultStore { get; } /// /// Gets a that specifies the that is used by @@ -38,9 +39,9 @@ public sealed class ReportingConfiguration public ChatConfiguration? ChatConfiguration { get; } /// - /// Gets the that should be used to cache AI responses. + /// Gets the that should be used to cache AI responses. /// - public IResponseCacheProvider? ResponseCacheProvider { get; } + public IEvaluationResponseCacheProvider? ResponseCacheProvider { get; } /// /// Gets the collection of unique strings that should be hashed when generating the cache keys for cached AI @@ -101,7 +102,7 @@ public sealed class ReportingConfiguration /// The set of s that should be invoked to evaluate AI responses. /// /// - /// The that should be used to persist the s. + /// The that should be used to persist the s. /// /// /// A that specifies the that is used by @@ -109,8 +110,8 @@ public sealed class ReportingConfiguration /// none of the included are AI-based. /// /// - /// The that should be used to cache AI responses. If omitted, AI responses - /// will not be cached. + /// The that should be used to cache AI responses. If omitted, AI + /// responses will not be cached. /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI @@ -134,9 +135,9 @@ public sealed class ReportingConfiguration #pragma warning disable S107 // Methods should not have too many parameters public ReportingConfiguration( IEnumerable evaluators, - IResultStore resultStore, + IEvaluationResultStore resultStore, ChatConfiguration? chatConfiguration = null, - IResponseCacheProvider? responseCacheProvider = null, + IEvaluationResponseCacheProvider? responseCacheProvider = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs index eb58685bf0c..5fa46e7e4ec 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs @@ -93,7 +93,7 @@ public sealed class ScenarioRun : IAsyncDisposable public ChatConfiguration? ChatConfiguration { get; } private readonly CompositeEvaluator _compositeEvaluator; - private readonly IResultStore _resultStore; + private readonly IEvaluationResultStore _resultStore; private readonly Func? _evaluationMetricInterpreter; private readonly ChatDetails? _chatDetails; private readonly IEnumerable? _tags; @@ -106,7 +106,7 @@ internal ScenarioRun( string iterationName, string executionName, IEnumerable evaluators, - IResultStore resultStore, + IEvaluationResultStore resultStore, ChatConfiguration? chatConfiguration = null, Func? evaluationMetricInterpreter = null, ChatDetails? chatDetails = null, @@ -189,7 +189,7 @@ await _compositeEvaluator.EvaluateAsync( /// /// Disposes the and writes the to the configured - /// . + /// . /// /// A that represents the asynchronous operation. public async ValueTask DisposeAsync() diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs index 3b723a2d258..08822f18d02 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs @@ -85,16 +85,11 @@ public static ValueTask EvaluateAsync( this ScenarioRun scenarioRun, ChatMessage modelResponse, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(scenarioRun); - - return scenarioRun.EvaluateAsync( - messages: [], + CancellationToken cancellationToken = default) => + scenarioRun.EvaluateAsync( new ChatResponse(modelResponse), additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an @@ -148,16 +143,12 @@ public static ValueTask EvaluateAsync( ChatMessage userRequest, ChatMessage modelResponse, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(scenarioRun); - - return scenarioRun.EvaluateAsync( - messages: [userRequest], + CancellationToken cancellationToken = default) => + scenarioRun.EvaluateAsync( + userRequest, new ChatResponse(modelResponse), additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs index 94ab92e177b..e967fdd1db9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs @@ -32,6 +32,10 @@ public static class DiskBasedReportingConfiguration /// /// to enable caching of AI responses; otherwise. /// + /// + /// An optional that specifies the maximum amount of time that cached AI responses should + /// survive in the cache before they are considered expired and evicted. + /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI /// responses. See for more information about this concept. @@ -61,6 +65,7 @@ public static ReportingConfiguration Create( IEnumerable evaluators, ChatConfiguration? chatConfiguration = null, bool enableResponseCaching = true, + TimeSpan? timeToLiveForCacheEntries = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, @@ -69,12 +74,12 @@ public static ReportingConfiguration Create( { storageRootPath = Path.GetFullPath(storageRootPath); - IResponseCacheProvider? responseCacheProvider = + IEvaluationResponseCacheProvider? responseCacheProvider = chatConfiguration is not null && enableResponseCaching - ? new DiskBasedResponseCacheProvider(storageRootPath) + ? new DiskBasedResponseCacheProvider(storageRootPath, timeToLiveForCacheEntries) : null; - IResultStore resultStore = new DiskBasedResultStore(storageRootPath); + IEvaluationResultStore resultStore = new DiskBasedResultStore(storageRootPath); return new ReportingConfiguration( evaluators, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs index feb75df1dba..8b60fe5a272 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs @@ -14,8 +14,9 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An that returns an that can cache AI responses -/// for a particular under the specified on disk. +/// An that returns an that can cache +/// AI responses for a particular under the specified on +/// disk. /// /// /// The path to a directory on disk under which the cached AI responses should be stored. @@ -26,15 +27,18 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// public sealed class DiskBasedResponseCacheProvider( string storageRootPath, - TimeSpan? timeToLiveForCacheEntries = null) : IResponseCacheProvider + TimeSpan? timeToLiveForCacheEntries = null) : IEvaluationResponseCacheProvider { private readonly Func _provideDateTime = () => DateTime.UtcNow; /// /// Intended for testing purposes only. /// - internal DiskBasedResponseCacheProvider(string storageRootPath, Func provideDateTime) - : this(storageRootPath) + internal DiskBasedResponseCacheProvider( + string storageRootPath, + Func provideDateTime, + TimeSpan? timeToLiveForCacheEntries = null) + : this(storageRootPath, timeToLiveForCacheEntries) { _provideDateTime = provideDateTime; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs index de1517dca99..4662857ec59 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs @@ -16,9 +16,9 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An implementation that stores s on disk. +/// An implementation that stores s on disk. /// -public sealed class DiskBasedResultStore : IResultStore +public sealed class DiskBasedResultStore : IEvaluationResultStore { private const string DeserializationFailedMessage = "Unable to deserialize the scenario run result file at {0}."; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs index 6ec3793d0da..0334d6aa08c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs @@ -4,6 +4,7 @@ using System; namespace Microsoft.Extensions.AI.Evaluation.Safety; + internal static class AIContentExtensions { internal static bool IsTextOrUsage(this AIContent content) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs index 428940955ff..b771dc008c8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs @@ -9,5 +9,5 @@ internal enum ContentSafetyServicePayloadFormat QuestionAnswer, QueryResponse, ContextCompletion, - Conversation, + Conversation } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs index 025ef58b809..1ba8ae270e6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs @@ -20,14 +20,14 @@ public enum EvaluationRating Inconclusive, /// - /// A value that indicates that the is interpreted as being exceptional. + /// A value that indicates that the is interpreted as being unacceptable. /// - Exceptional, + Unacceptable, /// - /// A value that indicates that the is interpreted as being good. + /// A value that indicates that the is interpreted as being poor. /// - Good, + Poor, /// /// A value that indicates that the is interpreted as being average. @@ -35,12 +35,12 @@ public enum EvaluationRating Average, /// - /// A value that indicates that the is interpreted as being poor. + /// A value that indicates that the is interpreted as being good. /// - Poor, + Good, /// - /// A value that indicates that the is interpreted as being unacceptable. + /// A value that indicates that the is interpreted as being exceptional. /// - Unacceptable, + Exceptional } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs index 3ffda8fb8f9..8d25085f4e0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs @@ -131,17 +131,12 @@ public static ValueTask EvaluateAsync( ChatMessage modelResponse, ChatConfiguration? chatConfiguration = null, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(evaluator); - - return evaluator.EvaluateAsync( - messages: [], + CancellationToken cancellationToken = default) => + evaluator.EvaluateAsync( new ChatResponse(modelResponse), chatConfiguration, additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an @@ -225,17 +220,13 @@ public static ValueTask EvaluateAsync( ChatMessage modelResponse, ChatConfiguration? chatConfiguration = null, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(evaluator); - - return evaluator.EvaluateAsync( - messages: [userRequest], + CancellationToken cancellationToken = default) => + evaluator.EvaluateAsync( + userRequest, new ChatResponse(modelResponse), chatConfiguration, additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs index 90f7a2b29aa..b56a2673b60 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs @@ -71,7 +71,7 @@ static QualityEvaluatorTests() DiskBasedReportingConfiguration.Create( storageRootPath: Settings.Current.StorageRootPath, evaluators: [groundednessEvaluator, equivalenceEvaluator, completenessEvaluator, retrievalEvaluator], - chatConfiguration, + chatConfiguration: chatConfiguration, executionName: Constants.Version, tags: [version, date, projectName, testClass, provider, model, temperature, usesContext]); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs index b135a64a04c..2f936621147 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs @@ -47,9 +47,9 @@ public async Task DisposeAsync() internal override bool IsConfigured => Settings.Current.Configured; - internal override IResponseCacheProvider CreateResponseCacheProvider() + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider() => new AzureStorageResponseCacheProvider(_dirClient!); - internal override IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) => new AzureStorageResponseCacheProvider(_dirClient!, provideDateTime); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs index 610f6345524..62163d5e681 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs @@ -47,7 +47,7 @@ public async Task DisposeAsync() public override bool IsConfigured => Settings.Current.Configured; - public override IResultStore CreateResultStore() + public override IEvaluationResultStore CreateResultStore() => new AzureStorageResultStore(_dirClient!); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs index e0ba0c171d1..8305fe8ddb3 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs @@ -45,9 +45,9 @@ public Task DisposeAsync() internal override bool IsConfigured => true; - internal override IResponseCacheProvider CreateResponseCacheProvider() + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider() => new DiskBasedResponseCacheProvider(UseTempStoragePath()); - internal override IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) => new DiskBasedResponseCacheProvider(UseTempStoragePath(), provideDateTime); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs index 1fee1b9996c..77cabfd7ffd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs @@ -44,7 +44,7 @@ public Task DisposeAsync() public override bool IsConfigured => true; - public override IResultStore CreateResultStore() + public override IEvaluationResultStore CreateResultStore() => new DiskBasedResultStore(UseTempStoragePath()); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs index 76e50244b92..b69014e631b 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs @@ -18,8 +18,8 @@ public abstract class ResponseCacheTester private static readonly string _keyB = "B Key"; private static readonly byte[] _responseB = Encoding.UTF8.GetBytes("Content B"); - internal abstract IResponseCacheProvider CreateResponseCacheProvider(); - internal abstract IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime); + internal abstract IEvaluationResponseCacheProvider CreateResponseCacheProvider(); + internal abstract IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime); internal abstract bool IsConfigured { get; } private void SkipIfNotConfigured() @@ -37,7 +37,7 @@ public async Task AddUncachedEntry() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(AddUncachedEntry), iterationName); Assert.NotNull(cache); @@ -58,7 +58,7 @@ public async Task RemoveCachedEntry() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -85,7 +85,7 @@ public async Task CacheEntryExpiration() DateTime now = DateTime.UtcNow; DateTime provideDateTime() => now; - IResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -106,7 +106,7 @@ public async Task MultipleCacheInstances() { SkipIfNotConfigured(); - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(MultipleCacheInstances), "Async"); Assert.NotNull(cache); IDistributedCache cache2 = await provider.GetCacheAsync(nameof(MultipleCacheInstances), "Async"); @@ -133,7 +133,7 @@ public async Task DeleteExpiredEntries() DateTime now = DateTime.UtcNow; DateTime provideDateTime() => now; - IResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -163,7 +163,7 @@ public async Task ResetCache() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs index 1ce033b3cd7..995b77a8c5e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs @@ -13,7 +13,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Tests; public abstract class ResultStoreTester { - public abstract IResultStore CreateResultStore(); + public abstract IEvaluationResultStore CreateResultStore(); public abstract bool IsConfigured { get; } @@ -39,7 +39,8 @@ private static ScenarioRunResult CreateTestResult(string scenarioName, string it private static string ScenarioName(int n) => $"Test.Scenario.{n}"; private static string IterationName(int n) => $"Iteration {n}"; - private static async Task> LoadResultsAsync(int n, IResultStore resultStore) + private static async Task> + LoadResultsAsync(int n, IEvaluationResultStore resultStore) { List<(string executionName, string scenarioName, string iterationName)> results = []; await foreach (string executionName in resultStore.GetLatestExecutionNamesAsync(n)) @@ -69,7 +70,7 @@ public async Task WriteAndReadResults() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string newExecutionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -108,7 +109,7 @@ public async Task WriteAndReadHistoricalResults() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string firstExecutionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -152,7 +153,7 @@ public async Task DeleteExecutions() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -176,7 +177,7 @@ public async Task DeleteSomeExecutions() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName0 = $"Test Execution {Path.GetRandomFileName()}"; @@ -211,7 +212,7 @@ public async Task DeleteScenarios() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -246,7 +247,7 @@ public async Task DeleteIterations() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; From 2d442e1277ee62ad3cb54c59c0b0a59655809f53 Mon Sep 17 00:00:00 2001 From: Shyam N Date: Fri, 9 May 2025 13:43:33 -0700 Subject: [PATCH 15/26] Allow image rendering in evaluation report (#6407) --- .../TypeScript/html-report/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html index c34388e543c..7f6e82be184 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html @@ -8,7 +8,7 @@ - + Codestin Search App From 7a1b9197dcbd4eeaf8fec623da1cb71b1cada4db Mon Sep 17 00:00:00 2001 From: Jeff Handley Date: Wed, 30 Apr 2025 19:16:54 -0700 Subject: [PATCH 16/26] Merge from release/9.4 into main, updating --- eng/Versions.props | 20 +++++++++---------- src/ProjectTemplates/GeneratedContent.targets | 3 ++- .../ChatWithCustomData-CSharp.Web/README.md | 10 ++++++++++ .../src/ChatWithCustomData/README.Aspire.md | 9 +++------ .../aichatweb/aichatweb.csproj | 2 +- .../aichatweb/README.md | 8 ++++++++ .../aichatweb.AppHost.csproj | 6 +++--- .../aichatweb.Web/aichatweb.Web.csproj | 6 +++--- .../aichatweb/aichatweb.csproj | 4 ++-- 9 files changed, 42 insertions(+), 26 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index b2dd32f628d..607e12c2c6d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -156,21 +156,21 @@ 4.8.0 3.3.4 - 9.1.0 - 9.1.0-preview.1.25121.10 + 9.2.1 + 9.2.1-preview.1.25222.1 1.0.0-beta.6 2.2.0-beta.4 1.13.2 11.6.0 - 9.3.1-beta.260 - 9.3.1-beta.260 - 9.3.1-beta.260 - 9.3.1-beta.260 + 9.4.1-beta.277 + 9.4.1-beta.277 + 9.4.1-beta.277 + 9.4.1-beta.277 9.2.0 - 1.45.0-preview - 1.45.0-preview - 1.45.0 - 5.1.12 + 1.47.0-preview + 1.47.0-preview + 1.47.0 + 5.1.13 1.9.0 0.1.9 6.0.1 diff --git a/src/ProjectTemplates/GeneratedContent.targets b/src/ProjectTemplates/GeneratedContent.targets index 4767af46a31..832d533e66a 100644 --- a/src/ProjectTemplates/GeneratedContent.targets +++ b/src/ProjectTemplates/GeneratedContent.targets @@ -21,10 +21,11 @@ false + false $(TemplatePinnedRepoPackagesVersion) - $(TemplatePinnedRepoAIPackagesVersion) + $(TemplatePinnedRepoAIPackagesVersion) $(TemplatePinnedMicrosoftEntityFrameworkCoreSqliteVersion) diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md index 44decea800c..37f10b83ce2 100644 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md @@ -9,6 +9,16 @@ This project is an AI chat application that demonstrates how to chat with custom ### Prerequisites To use Azure OpenAI or Azure AI Search, you need an Azure account. If you don't already have one, [create an Azure account](https://azure.microsoft.com/free/). +#### ---#endif +#### ---#if (IsOllama) +### Known Issues + +#### Errors running Ollama or Docker + +A recent incompatibility was found between Ollama and Docker Desktop. This issue results in runtime errors when connecting to Ollama, and the workaround for that can lead to Docker not working for Aspire projects. + +This incompatibility can be addressed by upgrading to Docker Desktop 4.41.1. See [ollama/ollama#9509](https://github.com/ollama/ollama/issues/9509#issuecomment-2842461831) for more information and a link to install the version of Docker Desktop with the fix. + #### ---#endif # Configure the AI Model Provider #### ---#if (IsGHModels) diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md index d1677b7ba78..f7c944dacc8 100644 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md @@ -10,17 +10,14 @@ This project is an AI chat application that demonstrates how to chat with custom To use Azure OpenAI or Azure AI Search, you need an Azure account. If you don't already have one, [create an Azure account](https://azure.microsoft.com/free/). #### ---#endif -#### ---#if (UseQdrant) ### Known Issues -#### Errors After Updating to Aspire Version 9.2.0 -This project is not currently compatible with Aspire 9.2.0, and all Aspire package versions are set to 9.1.0. Updating [Aspire.Qdrant.Client](https://www.nuget.org/packages/Aspire.Qdrant.Client) to version 9.2.0 causes an incompatibility with [Microsoft.SemanticKernel.Connectors.Qdrant](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Qdrant) where different versions of [Qdrant.Client](https://www.nuget.org/packages/Qdrant.Client) are required. Attempting to run the project with `Aspire.Qdrant.Client` version 9.2.0 will result in the following exception: +#### Errors running Ollama or Docker -> System.MissingMethodException: Method not found: 'Qdrant.Client.Grpc.Vectors Qdrant.Client.Grpc.ScoredPoint.get_Vectors()' +A recent incompatibility was found between Ollama and Docker Desktop. This issue results in runtime errors when connecting to Ollama, and the workaround for that can lead to Docker not working for Aspire projects. -Once a version of `Microsoft.SemanticKernel.Connectors.Qdrant` is published with a dependency on `Qdrant.Client` version `>= 1.13.0`, the Aspire packages can also be updated to version 9.2.0. +This incompatibility can be addressed by upgrading to Docker Desktop 4.41.1. See [ollama/ollama#9509](https://github.com/ollama/ollama/issues/9509#issuecomment-2842461831) for more information and a link to install the version of Docker Desktop with the fix. -#### ---#endif # Configure the AI Model Provider #### ---#if (IsGHModels) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj index 2d9ade69626..4b1fad034a9 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj @@ -11,7 +11,7 @@ - + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md index 0a467b898bd..c05c18281ef 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md @@ -5,6 +5,14 @@ This project is an AI chat application that demonstrates how to chat with custom >[!NOTE] > Before running this project you need to configure the API keys or endpoints for the providers you have chosen. See below for details specific to your choices. +### Known Issues + +#### Errors running Ollama or Docker + +A recent incompatibility was found between Ollama and Docker Desktop. This issue results in runtime errors when connecting to Ollama, and the workaround for that can lead to Docker not working for Aspire projects. + +This incompatibility can be addressed by upgrading to Docker Desktop 4.41.1. See [ollama/ollama#9509](https://github.com/ollama/ollama/issues/9509#issuecomment-2842461831) for more information and a link to install the version of Docker Desktop with the fix. + # Configure the AI Model Provider ## Using GitHub Models diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.AppHost/aichatweb.AppHost.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.AppHost/aichatweb.AppHost.csproj index 09439522151..a74ef7b7f3b 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.AppHost/aichatweb.AppHost.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.AppHost/aichatweb.AppHost.csproj @@ -1,6 +1,6 @@  - + Exe @@ -12,8 +12,8 @@ - - + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/aichatweb.Web.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/aichatweb.Web.csproj index 85c0c9cae35..c7b40c7b575 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/aichatweb.Web.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/aichatweb.Web.csproj @@ -8,11 +8,11 @@ - + - + - + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj index 93bebb61f9f..fd7131e492a 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj @@ -12,11 +12,11 @@ - + - + From 4ec4f85daeb97aa2629b6bb93f03cdfcb0c95ddb Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 9 May 2025 23:39:48 -0400 Subject: [PATCH 17/26] Move AIFunctionFactory down to M.E.AI.Abstractions (#6412) * Remove AIFunctionFactory dependency on M.E.DI This means reverting the recent changes to it that: - Special-cased KeyedServices - Special-cased IServiceProviderIsService - Used ActivatorUtilities.CreateInstance * Move AIFunctionFactory down to M.E.AI.Abstractions * Add CreateInstance delegate to AIFunctionFactoryOptions To enable use of ActivatorUtilities.CreateInstance or alternative. * Add some comments --- .../Functions/AIFunctionFactory.cs | 352 ++++++++---------- .../Functions/AIFunctionFactoryOptions.cs | 18 +- .../ChatClientStructuredOutputExtensions.cs | 26 +- .../Functions/AIFunctionFactory.Utilities.cs | 137 ------- .../Functions/AIFunctionFactoryTest.cs | 92 ++--- 5 files changed, 234 insertions(+), 391 deletions(-) rename src/Libraries/{Microsoft.Extensions.AI => Microsoft.Extensions.AI.Abstractions}/Functions/AIFunctionFactory.cs (82%) rename src/Libraries/{Microsoft.Extensions.AI => Microsoft.Extensions.AI.Abstractions}/Functions/AIFunctionFactoryOptions.cs (87%) delete mode 100644 src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.Utilities.cs diff --git a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs similarity index 82% rename from src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs index 4878239f35b..3f090a2ac3b 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs @@ -2,11 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.IO; #if !NET using System.Linq; #endif @@ -15,23 +17,26 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization.Metadata; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Shared.Collections; using Microsoft.Shared.Diagnostics; #pragma warning disable CA1031 // Do not catch general exception types +#pragma warning disable S2333 // Redundant modifiers should not be used #pragma warning disable S3011 // Reflection should not be used to increase accessibility of classes, methods, or fields -#pragma warning disable SA1118 // Parameter should not span multiple lines -#pragma warning disable SA1500 // Braces for multi-line statements should not share line namespace Microsoft.Extensions.AI; -/// Provides factory methods for creating commonly used implementations of . +/// Provides factory methods for creating commonly-used implementations of . /// Invoke .NET functions using an AI model. public static partial class AIFunctionFactory { + // NOTE: + // Unlike most library code, AIFunctionFactory uses ConfigureAwait(true) rather than ConfigureAwait(false). This is to + // enable AIFunctionFactory to be used with methods that might be context-aware, such as those employing a UI framework. + /// Holds the default options instance used when creating function. private static readonly AIFunctionFactoryOptions _defaultOptions = new(); @@ -71,25 +76,6 @@ public static partial class AIFunctionFactory /// . /// /// - /// - /// - /// By default, parameters attributed with are resolved from the - /// property and are not included in the JSON schema. If the parameter is optional, such that a default value is provided, - /// is allowed to be ; otherwise, - /// must be non-, or else the invocation will fail with an exception due to the required nature of the parameter. - /// The handling of such parameters may be overridden via . - /// - /// - /// - /// - /// When the is constructed, it may be passed an via - /// . Any parameter that can be satisfied by that - /// according to will not be included in the generated JSON schema and will be resolved - /// from the provided to via , - /// rather than from the argument collection. The handling of such parameters may be overridden via - /// . - /// - /// /// /// All other parameter types are, by default, bound from the dictionary passed into /// and are included in the generated JSON schema. This may be overridden by the provided @@ -170,23 +156,6 @@ public static AIFunction Create(Delegate method, AIFunctionFactoryOptions? optio /// optional or not. /// /// - /// - /// - /// By default, parameters attributed with are resolved from the - /// property and are not included in the JSON schema. If the parameter is optional, such that a default value is provided, - /// is allowed to be ; otherwise, - /// must be non-, or else the invocation will fail with an exception due to the required nature of the parameter. - /// - /// - /// - /// - /// When the is constructed, it may be passed an via - /// . Any parameter that can be satisfied by that - /// according to will not be included in the generated JSON schema and will be resolved - /// from the provided to via , - /// rather than from the argument collection. - /// - /// /// /// All other parameter types are bound from the dictionary passed into /// and are included in the generated JSON schema. @@ -270,25 +239,6 @@ public static AIFunction Create(Delegate method, string? name = null, string? de /// . /// /// - /// - /// - /// By default, parameters attributed with are resolved from the - /// property and are not included in the JSON schema. If the parameter is optional, such that a default value is provided, - /// is allowed to be ; otherwise, - /// must be non-, or else the invocation will fail with an exception due to the required nature of the parameter. - /// The handling of such parameters may be overridden via . - /// - /// - /// - /// - /// When the is constructed, it may be passed an via - /// . Any parameter that can be satisfied by that - /// according to will not be included in the generated JSON schema and will be resolved - /// from the provided to via , - /// rather than from the argument collection. The handling of such parameters may be overridden via - /// . - /// - /// /// /// All other parameter types are, by default, bound from the dictionary passed into /// and are included in the generated JSON schema. This may be overridden by the provided @@ -379,23 +329,6 @@ public static AIFunction Create(MethodInfo method, object? target, AIFunctionFac /// optional or not. /// /// - /// - /// - /// By default, parameters attributed with are resolved from the - /// property and are not included in the JSON schema. If the parameter is optional, such that a default value is provided, - /// is allowed to be ; otherwise, - /// must be non-, or else the invocation will fail with an exception due to the required nature of the parameter. - /// - /// - /// - /// When the is constructed, it may be passed an via - /// . Any parameter that can be satisfied by that - /// according to will not be included in the generated JSON schema and will be resolved - /// from the provided to via , - /// rather than from the argument collection. - /// - /// - /// /// /// All other parameter types are bound from the dictionary passed into /// and are included in the generated JSON schema. @@ -447,10 +380,9 @@ public static AIFunction Create(MethodInfo method, object? target, string? name /// The instance method to be represented via the created . /// /// The to construct an instance of on which to invoke when - /// the resulting is invoked. If is provided, - /// will be used to construct the instance using those services; otherwise, - /// is used, utilizing the type's public parameterless constructor. - /// If an instance can't be constructed, an exception is thrown during the function's invocation. + /// the resulting is invoked. is used, + /// utilizing the type's public parameterless constructor. If an instance can't be constructed, an exception is + /// thrown during the function's invocation. /// /// Metadata to use to override defaults inferred from . /// The created for invoking . @@ -494,25 +426,6 @@ public static AIFunction Create(MethodInfo method, object? target, string? name /// . /// /// - /// - /// - /// By default, parameters attributed with are resolved from the - /// property and are not included in the JSON schema. If the parameter is optional, such that a default value is provided, - /// is allowed to be ; otherwise, - /// must be non-, or else the invocation will fail with an exception due to the required nature of the parameter. - /// The handling of such parameters may be overridden via . - /// - /// - /// - /// - /// When the is constructed, it may be passed an via - /// . Any parameter that can be satisfied by that - /// according to will not be included in the generated JSON schema and will be resolved - /// from the provided to via , - /// rather than from the argument collection. The handling of such parameters may be overridden via - /// . - /// - /// /// /// All other parameter types are, by default, bound from the dictionary passed into /// and are included in the generated JSON schema. This may be overridden by the provided @@ -627,6 +540,7 @@ private ReflectionAIFunction( { FunctionDescriptor = functionDescriptor; TargetType = targetType; + CreateInstance = options.CreateInstance; AdditionalProperties = options.AdditionalProperties ?? EmptyReadOnlyDictionary.Instance; } @@ -634,6 +548,8 @@ private ReflectionAIFunction( public object? Target { get; } [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] public Type? TargetType { get; } + public Func? CreateInstance { get; } + public override IReadOnlyDictionary AdditionalProperties { get; } public override string Name => FunctionDescriptor.Name; public override string Description => FunctionDescriptor.Description; @@ -654,9 +570,14 @@ private ReflectionAIFunction( Debug.Assert(target is null, "Expected target to be null when we have a non-null target type"); Debug.Assert(!FunctionDescriptor.Method.IsStatic, "Expected an instance method"); - target = arguments.Services is { } services ? - ActivatorUtilities.CreateInstance(services, targetType!) : + target = CreateInstance is not null ? + CreateInstance(targetType, arguments) : Activator.CreateInstance(targetType); + if (target is null) + { + Throw.InvalidOperationException("Unable to create an instance of the target type."); + } + disposeTarget = true; } @@ -669,7 +590,7 @@ private ReflectionAIFunction( } return await FunctionDescriptor.ReturnParameterMarshaller( - ReflectionInvoke(FunctionDescriptor.Method, target, args), cancellationToken); + ReflectionInvoke(FunctionDescriptor.Method, target, args), cancellationToken).ConfigureAwait(true); } finally { @@ -677,7 +598,7 @@ private ReflectionAIFunction( { if (target is IAsyncDisposable ad) { - await ad.DisposeAsync(); + await ad.DisposeAsync().ConfigureAwait(true); } else if (target is IDisposable d) { @@ -709,7 +630,7 @@ public static ReflectionAIFunctionDescriptor GetOrCreate(MethodInfo method, AIFu serializerOptions.MakeReadOnly(); ConcurrentDictionary innerCache = _descriptorCache.GetOrCreateValue(serializerOptions); - DescriptorKey key = new(method, options.Name, options.Description, options.ConfigureParameterBinding, options.MarshalResult, options.Services, schemaOptions); + DescriptorKey key = new(method, options.Name, options.Description, options.ConfigureParameterBinding, options.MarshalResult, schemaOptions); if (innerCache.TryGetValue(key, out ReflectionAIFunctionDescriptor? descriptor)) { return descriptor; @@ -736,8 +657,6 @@ private ReflectionAIFunctionDescriptor(DescriptorKey key, JsonSerializerOptions } } - IServiceProviderIsService? serviceProviderIsService = key.Services?.GetService(); - // Use that binding information to impact the schema generation. AIJsonSchemaCreateOptions schemaOptions = key.SchemaOptions with { @@ -757,21 +676,6 @@ private ReflectionAIFunctionDescriptor(DescriptorKey key, JsonSerializerOptions return false; } - // If the parameter is attributed as [FromKeyedServices], exclude it, as we'll instead - // get its value from the IServiceProvider. - if (parameterInfo.GetCustomAttribute(inherit: true) is not null) - { - return false; - } - - // We assume that if the services used to create the function support a particular type, - // so too do the services that will be passed into InvokeAsync. This is the same basic assumption - // made in ASP.NET. - if (serviceProviderIsService?.IsService(parameterInfo.ParameterType) is true) - { - return false; - } - // If there was an existing IncludeParameter delegate, now defer to it as we've // excluded everything we need to exclude. if (key.SchemaOptions.IncludeParameter is { } existingIncludeParameter) @@ -793,7 +697,7 @@ private ReflectionAIFunctionDescriptor(DescriptorKey key, JsonSerializerOptions options = default; } - ParameterMarshallers[i] = GetParameterMarshaller(serializerOptions, options, parameters[i], serviceProviderIsService); + ParameterMarshallers[i] = GetParameterMarshaller(serializerOptions, options, parameters[i]); } // Get a marshaling delegate for the return value. @@ -863,8 +767,7 @@ static bool IsAsyncMethod(MethodInfo method) private static Func GetParameterMarshaller( JsonSerializerOptions serializerOptions, AIFunctionFactoryOptions.ParameterBindingOptions bindingOptions, - ParameterInfo parameter, - IServiceProviderIsService? serviceProviderIsService) + ParameterInfo parameter) { if (string.IsNullOrWhiteSpace(parameter.Name)) { @@ -911,56 +814,6 @@ static bool IsAsyncMethod(MethodInfo method) }; } - // For [FromKeyedServices] parameters, we resolve from the services passed to InvokeAsync via AIFunctionArguments. - if (parameter.GetCustomAttribute(inherit: true) is { } keyedAttr) - { - return (arguments, _) => - { - if ((arguments.Services as IKeyedServiceProvider)?.GetKeyedService(parameterType, keyedAttr.Key) is { } service) - { - return service; - } - - if (!parameter.HasDefaultValue) - { - if (arguments.Services is null) - { - ThrowNullServices(parameter.Name); - } - - Throw.ArgumentException(nameof(arguments), $"No service of type '{parameterType}' with key '{keyedAttr.Key}' was found for parameter '{parameter.Name}'."); - } - - return parameter.DefaultValue; - }; - } - - // For any parameters that are satisfiable from the IServiceProvider, we resolve from the services passed to InvokeAsync - // via AIFunctionArguments. This is determined by the same same IServiceProviderIsService instance used to determine whether - // the parameter should be included in the schema. - if (serviceProviderIsService?.IsService(parameterType) is true) - { - return (arguments, _) => - { - if (arguments.Services?.GetService(parameterType) is { } service) - { - return service; - } - - if (!parameter.HasDefaultValue) - { - if (arguments.Services is null) - { - ThrowNullServices(parameter.Name); - } - - Throw.ArgumentException(nameof(arguments), $"No service of type '{parameterType}' was found for parameter '{parameter.Name}'."); - } - - return parameter.DefaultValue; - }; - } - // For all other parameters, create a marshaller that tries to extract the value from the arguments dictionary. // Resolve the contract used to marshal the value from JSON -- can throw if not supported or not found. JsonTypeInfo? typeInfo = serializerOptions.GetTypeInfo(parameterType); @@ -1037,14 +890,14 @@ static void ThrowNullServices(string parameterName) => { return async (result, cancellationToken) => { - await ((Task)ThrowIfNullResult(result)); - return await marshalResult(null, null, cancellationToken); + await ((Task)ThrowIfNullResult(result)).ConfigureAwait(true); + return await marshalResult(null, null, cancellationToken).ConfigureAwait(true); }; } return async static (result, _) => { - await ((Task)ThrowIfNullResult(result)); + await ((Task)ThrowIfNullResult(result)).ConfigureAwait(true); return null; }; } @@ -1056,14 +909,14 @@ static void ThrowNullServices(string parameterName) => { return async (result, cancellationToken) => { - await ((ValueTask)ThrowIfNullResult(result)); - return await marshalResult(null, null, cancellationToken); + await ((ValueTask)ThrowIfNullResult(result)).ConfigureAwait(true); + return await marshalResult(null, null, cancellationToken).ConfigureAwait(true); }; } return async static (result, _) => { - await ((ValueTask)ThrowIfNullResult(result)); + await ((ValueTask)ThrowIfNullResult(result)).ConfigureAwait(true); return null; }; } @@ -1078,18 +931,18 @@ static void ThrowNullServices(string parameterName) => { return async (taskObj, cancellationToken) => { - await ((Task)ThrowIfNullResult(taskObj)); + await ((Task)ThrowIfNullResult(taskObj)).ConfigureAwait(true); object? result = ReflectionInvoke(taskResultGetter, taskObj, null); - return await marshalResult(result, taskResultGetter.ReturnType, cancellationToken); + return await marshalResult(result, taskResultGetter.ReturnType, cancellationToken).ConfigureAwait(true); }; } returnTypeInfo = serializerOptions.GetTypeInfo(taskResultGetter.ReturnType); return async (taskObj, cancellationToken) => { - await ((Task)ThrowIfNullResult(taskObj)); + await ((Task)ThrowIfNullResult(taskObj)).ConfigureAwait(true); object? result = ReflectionInvoke(taskResultGetter, taskObj, null); - return await SerializeResultAsync(result, returnTypeInfo, cancellationToken); + return await SerializeResultAsync(result, returnTypeInfo, cancellationToken).ConfigureAwait(true); }; } @@ -1104,9 +957,9 @@ static void ThrowNullServices(string parameterName) => return async (taskObj, cancellationToken) => { var task = (Task)ReflectionInvoke(valueTaskAsTask, ThrowIfNullResult(taskObj), null)!; - await task; + await task.ConfigureAwait(true); object? result = ReflectionInvoke(asTaskResultGetter, task, null); - return await marshalResult(result, asTaskResultGetter.ReturnType, cancellationToken); + return await marshalResult(result, asTaskResultGetter.ReturnType, cancellationToken).ConfigureAwait(true); }; } @@ -1114,9 +967,9 @@ static void ThrowNullServices(string parameterName) => return async (taskObj, cancellationToken) => { var task = (Task)ReflectionInvoke(valueTaskAsTask, ThrowIfNullResult(taskObj), null)!; - await task; + await task.ConfigureAwait(true); object? result = ReflectionInvoke(asTaskResultGetter, task, null); - return await SerializeResultAsync(result, returnTypeInfo, cancellationToken); + return await SerializeResultAsync(result, returnTypeInfo, cancellationToken).ConfigureAwait(true); }; } } @@ -1140,7 +993,7 @@ static void ThrowNullServices(string parameterName) => // Serialize asynchronously to support potential IAsyncEnumerable responses. using PooledMemoryStream stream = new(); - await JsonSerializer.SerializeAsync(stream, result, returnTypeInfo, cancellationToken); + await JsonSerializer.SerializeAsync(stream, result, returnTypeInfo, cancellationToken).ConfigureAwait(true); Utf8JsonReader reader = new(stream.GetBuffer()); return JsonElement.ParseValue(ref reader); } @@ -1169,7 +1022,126 @@ private record struct DescriptorKey( string? Description, Func? GetBindParameterOptions, Func>? MarshalResult, - IServiceProvider? Services, AIJsonSchemaCreateOptions SchemaOptions); } + + /// + /// Removes characters from a .NET member name that shouldn't be used in an AI function name. + /// + /// The .NET member name that should be sanitized. + /// + /// Replaces non-alphanumeric characters in the identifier with the underscore character. + /// Primarily intended to remove characters produced by compiler-generated method name mangling. + /// + private static string SanitizeMemberName(string memberName) => + InvalidNameCharsRegex().Replace(memberName, "_"); + + /// Regex that flags any character other than ASCII digits or letters or the underscore. +#if NET + [GeneratedRegex("[^0-9A-Za-z_]")] + private static partial Regex InvalidNameCharsRegex(); +#else + private static Regex InvalidNameCharsRegex() => _invalidNameCharsRegex; + private static readonly Regex _invalidNameCharsRegex = new("[^0-9A-Za-z_]", RegexOptions.Compiled); +#endif + + /// Invokes the MethodInfo with the specified target object and arguments. + private static object? ReflectionInvoke(MethodInfo method, object? target, object?[]? arguments) + { +#if NET + return method.Invoke(target, BindingFlags.DoNotWrapExceptions, binder: null, arguments, culture: null); +#else + try + { + return method.Invoke(target, BindingFlags.Default, binder: null, arguments, culture: null); + } + catch (TargetInvocationException e) when (e.InnerException is not null) + { + // If we're targeting .NET Framework, such that BindingFlags.DoNotWrapExceptions + // is ignored, the original exception will be wrapped in a TargetInvocationException. + // Unwrap it and throw that original exception, maintaining its stack information. + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e.InnerException).Throw(); + throw; + } +#endif + } + + /// + /// Implements a simple write-only memory stream that uses pooled buffers. + /// + private sealed class PooledMemoryStream : Stream + { + private const int DefaultBufferSize = 4096; + private byte[] _buffer; + private int _position; + + public PooledMemoryStream(int initialCapacity = DefaultBufferSize) + { + _buffer = ArrayPool.Shared.Rent(initialCapacity); + _position = 0; + } + + public ReadOnlySpan GetBuffer() => _buffer.AsSpan(0, _position); + public override bool CanWrite => true; + public override bool CanRead => false; + public override bool CanSeek => false; + public override long Length => _position; + public override long Position + { + get => _position; + set => throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + EnsureNotDisposed(); + EnsureCapacity(_position + count); + + Buffer.BlockCopy(buffer, offset, _buffer, _position, count); + _position += count; + } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (_buffer is not null) + { + ArrayPool.Shared.Return(_buffer); + _buffer = null!; + } + + base.Dispose(disposing); + } + + private void EnsureCapacity(int requiredCapacity) + { + if (requiredCapacity <= _buffer.Length) + { + return; + } + + int newCapacity = Math.Max(requiredCapacity, _buffer.Length * 2); + byte[] newBuffer = ArrayPool.Shared.Rent(newCapacity); + Buffer.BlockCopy(_buffer, 0, newBuffer, 0, _position); + + ArrayPool.Shared.Return(_buffer); + _buffer = newBuffer; + } + + private void EnsureNotDisposed() + { + if (_buffer is null) + { + Throw(); + static void Throw() => throw new ObjectDisposedException(nameof(PooledMemoryStream)); + } + } + } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactoryOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactoryOptions.cs similarity index 87% rename from src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactoryOptions.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactoryOptions.cs index 80c8b485c59..80ff394359d 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactoryOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactoryOptions.cs @@ -107,14 +107,22 @@ public AIFunctionFactoryOptions() public Func>? MarshalResult { get; set; } /// - /// Gets or sets optional services used in the construction of the . + /// Gets or sets a delegate used with to create the receiver instance. /// /// - /// These services will be used to determine which parameters should be satisifed from dependency injection. As such, - /// what services are satisfied via this provider should match what's satisfied via the provider passed into - /// via . + /// + /// creates instances that invoke an + /// instance method on the specified . This delegate is used to create the instance of the type that will be used to invoke the method. + /// By default if is , is used. If + /// is non-, the delegate is invoked with the to be instantiated and the + /// provided to the method. + /// + /// + /// Each created instance will be used for a single invocation. If the object is or , it will + /// be disposed of after the invocation completes. + /// /// - public IServiceProvider? Services { get; set; } + public Func? CreateInstance { get; set; } /// Provides configuration options produced by the delegate. public readonly record struct ParameterBindingOptions diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs index d7bc12a1a41..e35f8b87949 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs @@ -7,11 +7,13 @@ using System.Reflection; using System.Text.Json; using System.Text.Json.Nodes; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Shared.Diagnostics; #pragma warning disable SA1118 // Parameter should not span multiple lines +#pragma warning disable S2333 // Redundant modifiers should not be used namespace Microsoft.Extensions.AI; @@ -19,7 +21,7 @@ namespace Microsoft.Extensions.AI; /// Provides extension methods on that simplify working with structured output. /// /// Request a response with structured output. -public static class ChatClientStructuredOutputExtensions +public static partial class ChatClientStructuredOutputExtensions { private static readonly AIJsonSchemaCreateOptions _inferenceOptions = new() { @@ -197,7 +199,7 @@ public static async Task> GetResponseAsync( // the LLM backend is meant to do whatever's needed to explain the schema to the LLM. options.ResponseFormat = ChatResponseFormat.ForJsonSchema( schema, - schemaName: AIFunctionFactory.SanitizeMemberName(typeof(T).Name), + schemaName: SanitizeMemberName(typeof(T).Name), schemaDescription: typeof(T).GetCustomAttribute()?.Description); } else @@ -246,4 +248,24 @@ private static bool SchemaRepresentsObject(JsonElement schemaElement) _ => JsonValue.Create(element) }; } + + /// + /// Removes characters from a .NET member name that shouldn't be used in an AI function name. + /// + /// The .NET member name that should be sanitized. + /// + /// Replaces non-alphanumeric characters in the identifier with the underscore character. + /// Primarily intended to remove characters produced by compiler-generated method name mangling. + /// + private static string SanitizeMemberName(string memberName) => + InvalidNameCharsRegex().Replace(memberName, "_"); + + /// Regex that flags any character other than ASCII digits or letters or the underscore. +#if NET + [GeneratedRegex("[^0-9A-Za-z_]")] + private static partial Regex InvalidNameCharsRegex(); +#else + private static Regex InvalidNameCharsRegex() => _invalidNameCharsRegex; + private static readonly Regex _invalidNameCharsRegex = new("[^0-9A-Za-z_]", RegexOptions.Compiled); +#endif } diff --git a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.Utilities.cs b/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.Utilities.cs deleted file mode 100644 index cbafe78e5d3..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.Utilities.cs +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Buffers; -using System.IO; -using System.Reflection; -using System.Text.RegularExpressions; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI; - -public static partial class AIFunctionFactory -{ - /// - /// Removes characters from a .NET member name that shouldn't be used in an AI function name. - /// - /// The .NET member name that should be sanitized. - /// - /// Replaces non-alphanumeric characters in the identifier with the underscore character. - /// Primarily intended to remove characters produced by compiler-generated method name mangling. - /// - internal static string SanitizeMemberName(string memberName) - { - _ = Throw.IfNull(memberName); - return InvalidNameCharsRegex().Replace(memberName, "_"); - } - - /// Regex that flags any character other than ASCII digits or letters or the underscore. -#if NET - [GeneratedRegex("[^0-9A-Za-z_]")] - private static partial Regex InvalidNameCharsRegex(); -#else - private static Regex InvalidNameCharsRegex() => _invalidNameCharsRegex; - private static readonly Regex _invalidNameCharsRegex = new("[^0-9A-Za-z_]", RegexOptions.Compiled); -#endif - - /// Invokes the MethodInfo with the specified target object and arguments. - private static object? ReflectionInvoke(MethodInfo method, object? target, object?[]? arguments) - { -#if NET - return method.Invoke(target, BindingFlags.DoNotWrapExceptions, binder: null, arguments, culture: null); -#else - try - { - return method.Invoke(target, BindingFlags.Default, binder: null, arguments, culture: null); - } - catch (TargetInvocationException e) when (e.InnerException is not null) - { - // If we're targeting .NET Framework, such that BindingFlags.DoNotWrapExceptions - // is ignored, the original exception will be wrapped in a TargetInvocationException. - // Unwrap it and throw that original exception, maintaining its stack information. - System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e.InnerException).Throw(); - throw; - } -#endif - } - - /// - /// Implements a simple write-only memory stream that uses pooled buffers. - /// - private sealed class PooledMemoryStream : Stream - { - private const int DefaultBufferSize = 4096; - private byte[] _buffer; - private int _position; - - public PooledMemoryStream(int initialCapacity = DefaultBufferSize) - { - _buffer = ArrayPool.Shared.Rent(initialCapacity); - _position = 0; - } - - public ReadOnlySpan GetBuffer() => _buffer.AsSpan(0, _position); - public override bool CanWrite => true; - public override bool CanRead => false; - public override bool CanSeek => false; - public override long Length => _position; - public override long Position - { - get => _position; - set => throw new NotSupportedException(); - } - - public override void Write(byte[] buffer, int offset, int count) - { - EnsureNotDisposed(); - EnsureCapacity(_position + count); - - Buffer.BlockCopy(buffer, offset, _buffer, _position, count); - _position += count; - } - - public override void Flush() - { - } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - public override void SetLength(long value) => throw new NotSupportedException(); - - protected override void Dispose(bool disposing) - { - if (_buffer is not null) - { - ArrayPool.Shared.Return(_buffer); - _buffer = null!; - } - - base.Dispose(disposing); - } - - private void EnsureCapacity(int requiredCapacity) - { - if (requiredCapacity <= _buffer.Length) - { - return; - } - - int newCapacity = Math.Max(requiredCapacity, _buffer.Length * 2); - byte[] newBuffer = ArrayPool.Shared.Rent(newCapacity); - Buffer.BlockCopy(_buffer, 0, newBuffer, 0, _position); - - ArrayPool.Shared.Return(_buffer); - _buffer = newBuffer; - } - - private void EnsureNotDisposed() - { - if (_buffer is null) - { - Throw(); - static void Throw() => throw new ObjectDisposedException(nameof(PooledMemoryStream)); - } - } - } -} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs index 4b5ff9a0600..4f5037fc92d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs @@ -299,55 +299,6 @@ public async Task AIFunctionArguments_MissingServicesMayBeOptional() Assert.Equal("", result?.ToString()); } - [Fact] - public async Task IServiceProvider_ServicesInOptionsImpactsFunctionCreation() - { - ServiceCollection sc = new(); - sc.AddSingleton(new MyService(123)); - IServiceProvider sp = sc.BuildServiceProvider(); - - AIFunction func; - - // Services not provided to Create, non-optional argument - if (JsonSerializer.IsReflectionEnabledByDefault) - { - func = AIFunctionFactory.Create((MyService myService) => myService.Value); - Assert.Contains("myService", func.JsonSchema.ToString()); - await Assert.ThrowsAsync("arguments", () => func.InvokeAsync(new()).AsTask()); - await Assert.ThrowsAsync("arguments", () => func.InvokeAsync(new() { Services = sp }).AsTask()); - } - else - { - Assert.Throws(() => AIFunctionFactory.Create((MyService myService) => myService.Value)); - } - - // Services not provided to Create, optional argument - if (JsonSerializer.IsReflectionEnabledByDefault) - { - func = AIFunctionFactory.Create((MyService? myService = null) => myService?.Value ?? 456); - Assert.Contains("myService", func.JsonSchema.ToString()); - Assert.Contains("456", (await func.InvokeAsync(new()))?.ToString()); - Assert.Contains("456", (await func.InvokeAsync(new() { Services = sp }))?.ToString()); - } - else - { - Assert.Throws(() => AIFunctionFactory.Create((MyService myService) => myService.Value)); - } - - // Services provided to Create, non-optional argument - func = AIFunctionFactory.Create((MyService myService) => myService.Value, new() { Services = sp }); - Assert.DoesNotContain("myService", func.JsonSchema.ToString()); - await Assert.ThrowsAsync("arguments.Services", () => func.InvokeAsync(new()).AsTask()); - await Assert.ThrowsAsync("arguments", () => func.InvokeAsync(new() { Services = new ServiceCollection().BuildServiceProvider() }).AsTask()); - Assert.Contains("123", (await func.InvokeAsync(new() { Services = sp }))?.ToString()); - - // Services provided to Create, optional argument - func = AIFunctionFactory.Create((MyService? myService = null) => myService?.Value ?? 456, new() { Services = sp }); - Assert.DoesNotContain("myService", func.JsonSchema.ToString()); - Assert.Contains("456", (await func.InvokeAsync(new()))?.ToString()); - Assert.Contains("123", (await func.InvokeAsync(new() { Services = sp }))?.ToString()); - } - [Fact] public async Task Create_NoInstance_UsesActivatorUtilitiesWhenServicesAvailable() { @@ -364,6 +315,11 @@ public async Task Create_NoInstance_UsesActivatorUtilitiesWhenServicesAvailable( typeof(MyFunctionTypeWithOneArg), new() { + CreateInstance = (type, arguments) => + { + Assert.NotNull(arguments.Services); + return ActivatorUtilities.CreateInstance(arguments.Services, type); + }, MarshalResult = (result, type, cancellationToken) => new ValueTask(result), }); @@ -398,7 +354,7 @@ public async Task Create_NoInstance_ThrowsWhenCantConstructInstance() typeof(MyFunctionTypeWithOneArg)); Assert.NotNull(func); - await Assert.ThrowsAsync(async () => await func.InvokeAsync(new() { Services = sp })); + await Assert.ThrowsAsync(async () => await func.InvokeAsync(new() { Services = sp })); } [Fact] @@ -485,13 +441,13 @@ public async Task FromKeyedServices_ResolvesFromServiceProvider() sc.AddKeyedSingleton("key", service); IServiceProvider sp = sc.BuildServiceProvider(); - AIFunction f = AIFunctionFactory.Create(([FromKeyedServices("key")] MyService service, int myInteger) => service.Value + myInteger); + AIFunction f = AIFunctionFactory.Create(([FromKeyedServices("key")] MyService service, int myInteger) => service.Value + myInteger, + CreateKeyedServicesSupportOptions()); Assert.Contains("myInteger", f.JsonSchema.ToString()); Assert.DoesNotContain("service", f.JsonSchema.ToString()); - Exception e = await Assert.ThrowsAsync("arguments.Services", () => f.InvokeAsync(new() { ["myInteger"] = 1 }).AsTask()); - Assert.Contains("Services are required", e.Message); + Exception e = await Assert.ThrowsAsync("arguments.Services", () => f.InvokeAsync(new() { ["myInteger"] = 1 }).AsTask()); var result = await f.InvokeAsync(new() { ["myInteger"] = 1, Services = sp }); Assert.Contains("43", result?.ToString()); @@ -506,13 +462,13 @@ public async Task FromKeyedServices_NullKeysBindToNonKeyedServices() sc.AddSingleton(service); IServiceProvider sp = sc.BuildServiceProvider(); - AIFunction f = AIFunctionFactory.Create(([FromKeyedServices(null!)] MyService service, int myInteger) => service.Value + myInteger); + AIFunction f = AIFunctionFactory.Create(([FromKeyedServices(null!)] MyService service, int myInteger) => service.Value + myInteger, + CreateKeyedServicesSupportOptions()); Assert.Contains("myInteger", f.JsonSchema.ToString()); Assert.DoesNotContain("service", f.JsonSchema.ToString()); - Exception e = await Assert.ThrowsAsync("arguments.Services", () => f.InvokeAsync(new() { ["myInteger"] = 1 }).AsTask()); - Assert.Contains("Services are required", e.Message); + Exception e = await Assert.ThrowsAsync("arguments.Services", () => f.InvokeAsync(new() { ["myInteger"] = 1 }).AsTask()); var result = await f.InvokeAsync(new() { ["myInteger"] = 1, Services = sp }); Assert.Contains("43", result?.ToString()); @@ -528,7 +484,8 @@ public async Task FromKeyedServices_OptionalDefaultsToNull() IServiceProvider sp = sc.BuildServiceProvider(); AIFunction f = AIFunctionFactory.Create(([FromKeyedServices("key")] MyService? service = null, int myInteger = 0) => - service is null ? "null " + 1 : (service.Value + myInteger).ToString()); + service is null ? "null " + 1 : (service.Value + myInteger).ToString(), + CreateKeyedServicesSupportOptions()); Assert.Contains("myInteger", f.JsonSchema.ToString()); Assert.DoesNotContain("service", f.JsonSchema.ToString()); @@ -891,6 +848,27 @@ public StructWithDefaultCtor() } } + private static AIFunctionFactoryOptions CreateKeyedServicesSupportOptions() => + new AIFunctionFactoryOptions + { + ConfigureParameterBinding = p => + { + if (p.GetCustomAttribute() is { } attr) + { + return new() + { + BindParameter = (p, a) => + (a.Services as IKeyedServiceProvider)?.GetKeyedService(p.ParameterType, attr.Key) is { } s ? s : + p.HasDefaultValue ? p.DefaultValue : + throw new ArgumentException($"Unable to resolve argument for '{p.Name}'.", "arguments.Services"), + ExcludeFromSchema = true + }; + } + + return default; + }, + }; + [JsonSerializable(typeof(IAsyncEnumerable))] [JsonSerializable(typeof(int[]))] [JsonSerializable(typeof(string))] From 1ba107a0f3f00272a2354176086ebbb7576a7f0e Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sat, 10 May 2025 02:42:50 -0400 Subject: [PATCH 18/26] Fix handling of tool calls with some OpenAI endpoints (#6405) * Fix handling of tool calls with some endpoints Most assistant messages containing tool calls don't contain text as well (though some can). In such a case, we were still creating the assistant with empty text. While OpenAI's service permits that, some other endpoints are more finicky about it. This avoids doing so. * Reduce to single iteration through assistant content --- .../OpenAIChatClient.cs | 119 +++++++++++------- .../ChatClientIntegrationTests.cs | 6 +- .../OpenAIResponseClientIntegrationTests.cs | 2 + 3 files changed, 82 insertions(+), 45 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index c644bd77f21..98cf49fd696 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -157,30 +157,54 @@ void IDisposable.Dispose() } else if (input.Role == ChatRole.Assistant) { - AssistantChatMessage message = new(ToOpenAIChatContent(input.Contents)) - { - ParticipantName = input.AuthorName - }; - + List? contentParts = null; + List? toolCalls = null; + string? refusal = null; foreach (var content in input.Contents) { switch (content) { - case ErrorContent errorContent when errorContent.ErrorCode is nameof(message.Refusal): - message.Refusal = errorContent.Message; + case ErrorContent ec when ec.ErrorCode == nameof(AssistantChatMessage.Refusal): + refusal = ec.Message; break; - case FunctionCallContent callRequest: - message.ToolCalls.Add( - ChatToolCall.CreateFunctionToolCall( - callRequest.CallId, - callRequest.Name, - new(JsonSerializer.SerializeToUtf8Bytes( - callRequest.Arguments, - options.GetTypeInfo(typeof(IDictionary)))))); + case FunctionCallContent fc: + (toolCalls ??= []).Add( + ChatToolCall.CreateFunctionToolCall(fc.CallId, fc.Name, new(JsonSerializer.SerializeToUtf8Bytes( + fc.Arguments, options.GetTypeInfo(typeof(IDictionary)))))); break; + + default: + if (ToChatMessageContentPart(content) is { } part) + { + (contentParts ??= []).Add(part); + } + + break; + } + } + + AssistantChatMessage message; + if (contentParts is not null) + { + message = new(contentParts); + if (toolCalls is not null) + { + foreach (var toolCall in toolCalls) + { + message.ToolCalls.Add(toolCall); + } } } + else + { + message = toolCalls is not null ? + new(toolCalls) : + new(ChatMessageContentPart.CreateTextPart(string.Empty)); + } + + message.ParticipantName = input.AuthorName; + message.Refusal = refusal; yield return message; } @@ -191,38 +215,12 @@ void IDisposable.Dispose() private static List ToOpenAIChatContent(IList contents) { List parts = []; + foreach (var content in contents) { - switch (content) + if (ToChatMessageContentPart(content) is { } part) { - case TextContent textContent: - parts.Add(ChatMessageContentPart.CreateTextPart(textContent.Text)); - break; - - case UriContent uriContent when uriContent.HasTopLevelMediaType("image"): - parts.Add(ChatMessageContentPart.CreateImagePart(uriContent.Uri, GetImageDetail(content))); - break; - - case DataContent dataContent when dataContent.HasTopLevelMediaType("image"): - parts.Add(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(content))); - break; - - case DataContent dataContent when dataContent.HasTopLevelMediaType("audio"): - var audioData = BinaryData.FromBytes(dataContent.Data); - if (dataContent.MediaType.Equals("audio/mpeg", StringComparison.OrdinalIgnoreCase)) - { - parts.Add(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Mp3)); - } - else if (dataContent.MediaType.Equals("audio/wav", StringComparison.OrdinalIgnoreCase)) - { - parts.Add(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav)); - } - - break; - - case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase): - parts.Add(ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf")); - break; + parts.Add(part); } } @@ -234,6 +232,39 @@ private static List ToOpenAIChatContent(IList return parts; } + private static ChatMessageContentPart? ToChatMessageContentPart(AIContent content) + { + switch (content) + { + case TextContent textContent: + return ChatMessageContentPart.CreateTextPart(textContent.Text); + + case UriContent uriContent when uriContent.HasTopLevelMediaType("image"): + return ChatMessageContentPart.CreateImagePart(uriContent.Uri, GetImageDetail(content)); + + case DataContent dataContent when dataContent.HasTopLevelMediaType("image"): + return ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(content)); + + case DataContent dataContent when dataContent.HasTopLevelMediaType("audio"): + var audioData = BinaryData.FromBytes(dataContent.Data); + if (dataContent.MediaType.Equals("audio/mpeg", StringComparison.OrdinalIgnoreCase)) + { + return ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Mp3); + } + else if (dataContent.MediaType.Equals("audio/wav", StringComparison.OrdinalIgnoreCase)) + { + return ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav); + } + + break; + + case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase): + return ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf"); + } + + return null; + } + private static ChatImageDetailLevel? GetImageDetail(AIContent content) { if (content.AdditionalProperties?.TryGetValue("detail", out object? value) is true) diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs index edb6c5dd14c..994fef47517 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs @@ -77,7 +77,9 @@ public virtual async Task GetResponseAsync_WithEmptyMessage() var response = await _chatClient.GetResponseAsync( [ + new(ChatRole.System, []), new(ChatRole.User, []), + new(ChatRole.Assistant, []), new(ChatRole.User, "What is 1 + 2? Reply with a single number."), ]); @@ -618,9 +620,11 @@ public virtual async Task Caching_AfterFunctionInvocation_FunctionOutputUnchange var secondResponse = await chatClient.GetResponseAsync([message]); Assert.Equal(response.Text, secondResponse.Text); Assert.Equal(2, functionCallCount); - Assert.Equal(2, llmCallCount!.CallCount); + Assert.Equal(FunctionInvokingChatClientSetsConversationId ? 3 : 2, llmCallCount!.CallCount); } + public virtual bool FunctionInvokingChatClientSetsConversationId => false; + [ConditionalFact] public virtual async Task Caching_AfterFunctionInvocation_FunctionOutputChangedAsync() { diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs index bbfad1c571d..2c1d6cdc80e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs @@ -9,4 +9,6 @@ public class OpenAIResponseClientIntegrationTests : ChatClientIntegrationTests IntegrationTestHelpers.GetOpenAIClient() ?.GetOpenAIResponseClient(TestRunnerConfiguration.Instance["OpenAI:ChatModel"] ?? "gpt-4o-mini") .AsIChatClient(); + + public override bool FunctionInvokingChatClientSetsConversationId => true; } From 0fe91fac16eaa66623bb1a49b0c315ea2986fbf6 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 12 May 2025 06:24:48 -0400 Subject: [PATCH 19/26] Add missing [DebuggerDisplay] on AIFunctionArguments (#6422) --- .../Functions/AIFunctionArguments.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionArguments.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionArguments.cs index 8fa34e52c08..3238b88e532 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionArguments.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionArguments.cs @@ -4,6 +4,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; #pragma warning disable SA1111 // Closing parenthesis should be on line of last parameter #pragma warning disable SA1112 // Closing parenthesis should be on line of opening parenthesis @@ -24,6 +25,7 @@ namespace Microsoft.Extensions.AI; /// an if it needs to resolve any services from a dependency injection /// container. /// +[DebuggerDisplay("Count = {Count}")] public class AIFunctionArguments : IDictionary, IReadOnlyDictionary { /// The nominal arguments. From 43adf49802b785c382b9ac7fd968dd7b2c9a9a11 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 12 May 2025 07:05:11 -0400 Subject: [PATCH 20/26] Add WriteAsync overrides to stream helper in AIFunctionFactory (#6419) We use JsonSerializer.SerializeAsync but were missing the async overrides. As with MemoryStream, these don't need to queue. --- .../Functions/AIFunctionFactory.cs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs index 3f090a2ac3b..6534e041a7c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs @@ -26,6 +26,7 @@ #pragma warning disable CA1031 // Do not catch general exception types #pragma warning disable S2333 // Redundant modifiers should not be used #pragma warning disable S3011 // Reflection should not be used to increase accessibility of classes, methods, or fields +#pragma warning disable SA1202 // Public members should come before private members namespace Microsoft.Extensions.AI; @@ -1105,6 +1106,34 @@ public override void Flush() { } + public override Task FlushAsync(CancellationToken cancellationToken) => + Task.CompletedTask; + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + WriteAsync(new ReadOnlyMemory(buffer, offset, count), cancellationToken).AsTask(); + +#if NET + public override +#else + private +#endif + ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + EnsureNotDisposed(); + + if (cancellationToken.IsCancellationRequested) + { + return new ValueTask(Task.FromCanceled(cancellationToken)); + } + + EnsureCapacity(_position + buffer.Length); + + buffer.Span.CopyTo(_buffer.AsSpan(_position)); + _position += buffer.Length; + + return default; + } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); public override void SetLength(long value) => throw new NotSupportedException(); From 3e6df02ac2dead196974a8829b3bb5caa24afb0e Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 12 May 2025 07:16:58 -0400 Subject: [PATCH 21/26] Update CHANGELOGs for M.E.AI (#6416) * Update CHANGELOGs for M.E.AI --- .../Microsoft.Extensions.AI.Abstractions/CHANGELOG.md | 9 +++++++++ .../CHANGELOG.md | 5 +++++ .../Microsoft.Extensions.AI.Ollama/CHANGELOG.md | 4 ++++ .../Microsoft.Extensions.AI.OpenAI/CHANGELOG.md | 6 ++++++ src/Libraries/Microsoft.Extensions.AI/CHANGELOG.md | 6 ++++++ 5 files changed, 30 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md index c6639273d70..b4fe9d69a66 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md @@ -1,5 +1,14 @@ # Release History +## 9.4.4-preview.1.25259.16 + +- Added `AIJsonUtilities.TransformSchema` and supporting types. +- Added `BinaryEmbedding` for bit embeddings. +- Added `ChatOptions.RawRepresentationFactory` to make it easier to pass options to the underlying service. +- Added `Base64Data` property to `DataContent`. +- Moved `AIFunctionFactory` to `Microsoft.Extensions.AI.Abstractions`. +- Fixed `AIFunctionFactory` handling of default struct arguments. + ## 9.4.3-preview.1.25230.7 - Renamed `ChatThreadId` to `ConversationId` on `ChatResponse`, `ChatResponseUpdate`, and `ChatOptions`. diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md index aaf1ac1c67c..aeb023efae5 100644 --- a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md @@ -1,5 +1,10 @@ # Release History +## 9.4.4-preview.1.25259.16 + +- Added an `AsIEmbeddingGenerator` extension method for `ImageEmbeddingsClient`. +- Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. + ## 9.4.3-preview.1.25230.7 - Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. diff --git a/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md index 8822f8ddaea..e90fed2cdba 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History +## 9.4.4-preview.1.25259.16 + +- Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. + ## 9.4.3-preview.1.25230.7 - Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md index 05130ba3847..ad915d06aa7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 9.4.4-preview.1.25259.16 + +- Made `IChatClient` implementation more resilient with non-OpenAI services. +- Added `ErrorContent` to represent refusals. +- Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. + ## 9.4.3-preview.1.25230.7 - Reverted previous change that enabled `strict` schemas by default. diff --git a/src/Libraries/Microsoft.Extensions.AI/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI/CHANGELOG.md index 69cf9f12c46..25c15aed0d2 100644 --- a/src/Libraries/Microsoft.Extensions.AI/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 9.4.4-preview.1.25259.16 + +- Fixed `CachingChatClient` to avoid caching when `ConversationId` is set. +- Renamed `useJsonSchema` parameter in `GetResponseAsync` to `useJsonSchemaResponseFormat`. +- Updated `OpenTelemetryChatClient` and `OpenTelemetryEmbeddingGenerator` to conform to the latest 1.33.0 draft specification of the Semantic Conventions for Generative AI systems. + ## 9.4.3-preview.1.25230.7 - Updated the diagnostic spans emitted by `FunctionInvokingChatClient` to include total input and output token counts. From f49a64a23ffa5b71f701359baf2a86e3b4d4de92 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 12 May 2025 10:55:24 -0400 Subject: [PATCH 22/26] Replace Type targetType AIFunctionFactory.Create parameter with a func (#6424) --- .../Functions/AIFunctionFactory.cs | 56 +++++++----------- .../Functions/AIFunctionFactoryOptions.cs | 18 ------ .../Functions/AIFunctionFactoryTest.cs | 57 ++++++------------- 3 files changed, 37 insertions(+), 94 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs index 6534e041a7c..d5274186645 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs @@ -7,7 +7,6 @@ using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.IO; #if !NET using System.Linq; @@ -374,16 +373,15 @@ public static AIFunction Create(MethodInfo method, object? target, string? name } /// - /// Creates an instance for a method, specified via an for - /// and instance method, along with a representing the type of the target object to - /// instantiate each time the method is invoked. + /// Creates an instance for a method, specified via a for + /// an instance method and a for constructing an instance of + /// the receiver object each time the is invoked. /// /// The instance method to be represented via the created . - /// - /// The to construct an instance of on which to invoke when - /// the resulting is invoked. is used, - /// utilizing the type's public parameterless constructor. If an instance can't be constructed, an exception is - /// thrown during the function's invocation. + /// + /// Callback used on each function invocation to create an instance of the type on which the instance method + /// will be invoked. If the returned instance is or , it will be disposed of + /// after completes its invocation. /// /// Metadata to use to override defaults inferred from . /// The created for invoking . @@ -457,22 +455,16 @@ public static AIFunction Create(MethodInfo method, object? target, string? name /// /// /// is . - /// is . + /// is . /// represents a static method. /// represents an open generic method. /// contains a parameter without a parameter name. - /// is not assignable to 's declaring type. /// A parameter to or its return type is not serializable. public static AIFunction Create( MethodInfo method, - [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] Type targetType, - AIFunctionFactoryOptions? options = null) - { - _ = Throw.IfNull(method); - _ = Throw.IfNull(targetType); - - return ReflectionAIFunction.Build(method, targetType, options ?? _defaultOptions); - } + Func createInstanceFunc, + AIFunctionFactoryOptions? options = null) => + ReflectionAIFunction.Build(method, createInstanceFunc, options ?? _defaultOptions); private sealed class ReflectionAIFunction : AIFunction { @@ -503,10 +495,11 @@ public static ReflectionAIFunction Build(MethodInfo method, object? target, AIFu public static ReflectionAIFunction Build( MethodInfo method, - [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] Type targetType, + Func createInstanceFunc, AIFunctionFactoryOptions options) { _ = Throw.IfNull(method); + _ = Throw.IfNull(createInstanceFunc); if (method.ContainsGenericParameters) { @@ -518,13 +511,7 @@ public static ReflectionAIFunction Build( Throw.ArgumentException(nameof(method), "The method must be an instance method."); } - if (method.DeclaringType is { } declaringType && - !declaringType.IsAssignableFrom(targetType)) - { - Throw.ArgumentException(nameof(targetType), "The target type must be assignable to the method's declaring type."); - } - - return new(ReflectionAIFunctionDescriptor.GetOrCreate(method, options), targetType, options); + return new(ReflectionAIFunctionDescriptor.GetOrCreate(method, options), createInstanceFunc, options); } private ReflectionAIFunction(ReflectionAIFunctionDescriptor functionDescriptor, object? target, AIFunctionFactoryOptions options) @@ -536,20 +523,17 @@ private ReflectionAIFunction(ReflectionAIFunctionDescriptor functionDescriptor, private ReflectionAIFunction( ReflectionAIFunctionDescriptor functionDescriptor, - [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] Type targetType, + Func createInstanceFunc, AIFunctionFactoryOptions options) { FunctionDescriptor = functionDescriptor; - TargetType = targetType; - CreateInstance = options.CreateInstance; + CreateInstanceFunc = createInstanceFunc; AdditionalProperties = options.AdditionalProperties ?? EmptyReadOnlyDictionary.Instance; } public ReflectionAIFunctionDescriptor FunctionDescriptor { get; } public object? Target { get; } - [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] - public Type? TargetType { get; } - public Func? CreateInstance { get; } + public Func? CreateInstanceFunc { get; } public override IReadOnlyDictionary AdditionalProperties { get; } public override string Name => FunctionDescriptor.Name; @@ -566,14 +550,12 @@ private ReflectionAIFunction( object? target = Target; try { - if (TargetType is { } targetType) + if (CreateInstanceFunc is { } func) { Debug.Assert(target is null, "Expected target to be null when we have a non-null target type"); Debug.Assert(!FunctionDescriptor.Method.IsStatic, "Expected an instance method"); - target = CreateInstance is not null ? - CreateInstance(targetType, arguments) : - Activator.CreateInstance(targetType); + target = func(arguments); if (target is null) { Throw.InvalidOperationException("Unable to create an instance of the target type."); diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactoryOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactoryOptions.cs index 80ff394359d..e71a4687422 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactoryOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactoryOptions.cs @@ -106,24 +106,6 @@ public AIFunctionFactoryOptions() /// public Func>? MarshalResult { get; set; } - /// - /// Gets or sets a delegate used with to create the receiver instance. - /// - /// - /// - /// creates instances that invoke an - /// instance method on the specified . This delegate is used to create the instance of the type that will be used to invoke the method. - /// By default if is , is used. If - /// is non-, the delegate is invoked with the to be instantiated and the - /// provided to the method. - /// - /// - /// Each created instance will be used for a single invocation. If the object is or , it will - /// be disposed of after the invocation completes. - /// - /// - public Func? CreateInstance { get; set; } - /// Provides configuration options produced by the delegate. public readonly record struct ParameterBindingOptions { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs index 4f5037fc92d..6d448efb710 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs @@ -29,7 +29,8 @@ public void InvalidArguments_Throw() Assert.Throws("method", () => AIFunctionFactory.Create(method: null!, target: new object())); Assert.Throws("method", () => AIFunctionFactory.Create(method: null!, target: new object(), name: "myAiFunk")); Assert.Throws("target", () => AIFunctionFactory.Create(typeof(AIFunctionFactoryTest).GetMethod(nameof(InvalidArguments_Throw))!, (object?)null)); - Assert.Throws("targetType", () => AIFunctionFactory.Create(typeof(AIFunctionFactoryTest).GetMethod(nameof(InvalidArguments_Throw))!, (Type)null!)); + Assert.Throws("createInstanceFunc", () => + AIFunctionFactory.Create(typeof(AIFunctionFactoryTest).GetMethod(nameof(InvalidArguments_Throw))!, (Func)null!)); Assert.Throws("method", () => AIFunctionFactory.Create(typeof(List<>).GetMethod("Add")!, new List())); } @@ -312,16 +313,12 @@ public async Task Create_NoInstance_UsesActivatorUtilitiesWhenServicesAvailable( AIFunction func = AIFunctionFactory.Create( typeof(MyFunctionTypeWithOneArg).GetMethod(nameof(MyFunctionTypeWithOneArg.InstanceMethod))!, - typeof(MyFunctionTypeWithOneArg), - new() + static arguments => { - CreateInstance = (type, arguments) => - { - Assert.NotNull(arguments.Services); - return ActivatorUtilities.CreateInstance(arguments.Services, type); - }, - MarshalResult = (result, type, cancellationToken) => new ValueTask(result), - }); + Assert.NotNull(arguments.Services); + return ActivatorUtilities.CreateInstance(arguments.Services, typeof(MyFunctionTypeWithOneArg)); + }, + new() { MarshalResult = (result, type, cancellationToken) => new ValueTask(result) }); Assert.NotNull(func); var result = (Tuple?)await func.InvokeAsync(new() { Services = sp }); @@ -330,31 +327,25 @@ public async Task Create_NoInstance_UsesActivatorUtilitiesWhenServicesAvailable( } [Fact] - public async Task Create_NoInstance_UsesActivatorWhenServicesUnavailable() + public async Task Create_CreateInstanceReturnsNull_ThrowsDuringInvocation() { AIFunction func = AIFunctionFactory.Create( - typeof(MyFunctionTypeWithNoArgs).GetMethod(nameof(MyFunctionTypeWithNoArgs.InstanceMethod))!, - typeof(MyFunctionTypeWithNoArgs), - new() - { - MarshalResult = (result, type, cancellationToken) => new ValueTask(result), - }); + typeof(MyFunctionTypeWithOneArg).GetMethod(nameof(MyFunctionTypeWithOneArg.InstanceMethod))!, + static _ => null!); Assert.NotNull(func); - Assert.Equal("42", await func.InvokeAsync()); + await Assert.ThrowsAsync(async () => await func.InvokeAsync()); } [Fact] - public async Task Create_NoInstance_ThrowsWhenCantConstructInstance() + public async Task Create_WrongConstructedType_ThrowsDuringInvocation() { - var sp = new ServiceCollection().BuildServiceProvider(); - AIFunction func = AIFunctionFactory.Create( typeof(MyFunctionTypeWithOneArg).GetMethod(nameof(MyFunctionTypeWithOneArg.InstanceMethod))!, - typeof(MyFunctionTypeWithOneArg)); + static _ => new MyFunctionTypeWithNoArgs()); Assert.NotNull(func); - await Assert.ThrowsAsync(async () => await func.InvokeAsync(new() { Services = sp })); + await Assert.ThrowsAsync(async () => await func.InvokeAsync()); } [Fact] @@ -362,15 +353,7 @@ public void Create_NoInstance_ThrowsForStaticMethod() { Assert.Throws("method", () => AIFunctionFactory.Create( typeof(MyFunctionTypeWithNoArgs).GetMethod(nameof(MyFunctionTypeWithNoArgs.StaticMethod))!, - typeof(MyFunctionTypeWithNoArgs))); - } - - [Fact] - public void Create_NoInstance_ThrowsForMismatchedMethod() - { - Assert.Throws("targetType", () => AIFunctionFactory.Create( - typeof(MyFunctionTypeWithNoArgs).GetMethod(nameof(MyFunctionTypeWithNoArgs.InstanceMethod))!, - typeof(MyFunctionTypeWithOneArg))); + static _ => new MyFunctionTypeWithNoArgs())); } [Fact] @@ -378,7 +361,7 @@ public async Task Create_NoInstance_DisposableInstanceCreatedDisposedEachInvocat { AIFunction func = AIFunctionFactory.Create( typeof(DisposableService).GetMethod(nameof(DisposableService.GetThis))!, - typeof(DisposableService), + static _ => new DisposableService(), new() { MarshalResult = (result, type, cancellationToken) => new ValueTask(result), @@ -397,7 +380,7 @@ public async Task Create_NoInstance_AsyncDisposableInstanceCreatedDisposedEachIn { AIFunction func = AIFunctionFactory.Create( typeof(AsyncDisposableService).GetMethod(nameof(AsyncDisposableService.GetThis))!, - typeof(AsyncDisposableService), + static _ => new AsyncDisposableService(), new() { MarshalResult = (result, type, cancellationToken) => new ValueTask(result), @@ -416,7 +399,7 @@ public async Task Create_NoInstance_DisposableAndAsyncDisposableInstanceCreatedD { AIFunction func = AIFunctionFactory.Create( typeof(DisposableAndAsyncDisposableService).GetMethod(nameof(DisposableAndAsyncDisposableService.GetThis))!, - typeof(DisposableAndAsyncDisposableService), + static _ => new DisposableAndAsyncDisposableService(), new() { MarshalResult = (result, type, cancellationToken) => new ValueTask(result), @@ -821,11 +804,7 @@ public ValueTask DisposeAsync() private sealed class MyFunctionTypeWithNoArgs { - private string _value = "42"; - public static void StaticMethod() => throw new NotSupportedException(); - - public string InstanceMethod() => _value; } private sealed class MyFunctionTypeWithOneArg(MyArgumentType arg) From 57d0da30da8a8a9b8240a381701dee15752a35c1 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 12 May 2025 12:02:17 -0400 Subject: [PATCH 23/26] Remove debug-level logging of updates in LoggingChatClient (#6425) --- .../ChatCompletion/LoggingChatClient.cs | 14 ++------------ .../ChatCompletion/LoggingChatClientTests.cs | 2 -- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs index 3937d5db59b..aec72eddcdc 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs @@ -153,16 +153,9 @@ public override async IAsyncEnumerable GetStreamingResponseA throw; } - if (_logger.IsEnabled(LogLevel.Debug)) + if (_logger.IsEnabled(LogLevel.Trace)) { - if (_logger.IsEnabled(LogLevel.Trace)) - { - LogStreamingUpdateSensitive(AsJson(update)); - } - else - { - LogStreamingUpdate(); - } + LogStreamingUpdateSensitive(AsJson(update)); } yield return update; @@ -190,9 +183,6 @@ public override async IAsyncEnumerable GetStreamingResponseA [LoggerMessage(LogLevel.Trace, "{MethodName} completed: {ChatResponse}.")] private partial void LogCompletedSensitive(string methodName, string chatResponse); - [LoggerMessage(LogLevel.Debug, "GetStreamingResponseAsync received update.")] - private partial void LogStreamingUpdate(); - [LoggerMessage(LogLevel.Trace, "GetStreamingResponseAsync received update: {ChatResponseUpdate}")] private partial void LogStreamingUpdateSensitive(string chatResponseUpdate); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/LoggingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/LoggingChatClientTests.cs index 51638d1a252..cd383381c06 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/LoggingChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/LoggingChatClientTests.cs @@ -134,8 +134,6 @@ static async IAsyncEnumerable GetUpdatesAsync() { Assert.Collection(logs, entry => Assert.True(entry.Message.Contains("GetStreamingResponseAsync invoked.") && !entry.Message.Contains("biggest animal")), - entry => Assert.True(entry.Message.Contains("GetStreamingResponseAsync received update.") && !entry.Message.Contains("blue")), - entry => Assert.True(entry.Message.Contains("GetStreamingResponseAsync received update.") && !entry.Message.Contains("whale")), entry => Assert.Contains("GetStreamingResponseAsync completed.", entry.Message)); } else From 70a88ccf12a4f877bb924842592dd21624ea041b Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Mon, 12 May 2025 23:14:11 +0300 Subject: [PATCH 24/26] Add an AIJsonSchemaTransformOptions property inside AIJsonSchemaCreateOptions and mark redundant properties in the latter as obsolete. (#6427) * Add an AIJsonSchemaTransformOptions property inside AIJsonSchemaCreateOptions and mark redundant properties in the latter as obsolete. * s/inferred/created --- .../Utilities/AIJsonSchemaCreateOptions.cs | 17 ++- .../Utilities/AIJsonSchemaTransformOptions.cs | 5 + .../Utilities/AIJsonUtilities.Defaults.cs | 7 ++ .../AIJsonUtilities.Schema.Create.cs | 107 +++++------------- .../AIJsonUtilities.Schema.Transform.cs | 21 +++- .../AzureAIInferenceChatClient.cs | 3 +- .../OpenAIChatClient.cs | 3 +- .../ChatClientStructuredOutputExtensions.cs | 9 +- .../Utilities/AIJsonUtilitiesTests.cs | 77 ++++++++++--- ...atClientStructuredOutputExtensionsTests.cs | 34 +++++- 10 files changed, 175 insertions(+), 108 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaCreateOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaCreateOptions.cs index 8e3269f7a9c..8c53938f481 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaCreateOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaCreateOptions.cs @@ -38,22 +38,33 @@ public sealed record class AIJsonSchemaCreateOptions public Func? IncludeParameter { get; init; } /// - /// Gets a value indicating whether to include the type keyword in inferred schemas for .NET enums. + /// Gets a governing transformations on the JSON schema after it has been generated. /// + public AIJsonSchemaTransformOptions? TransformOptions { get; init; } + + /// + /// Gets a value indicating whether to include the type keyword in created schemas for .NET enums. + /// + [Obsolete("This property has been deprecated.")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] public bool IncludeTypeInEnumSchemas { get; init; } = true; /// /// Gets a value indicating whether to generate schemas with the additionalProperties set to false for .NET objects. /// - public bool DisallowAdditionalProperties { get; init; } = true; + [Obsolete("This property has been deprecated. Use the equivalent property in TransformOptions instead.")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + public bool DisallowAdditionalProperties { get; init; } /// - /// Gets a value indicating whether to include the $schema keyword in inferred schemas. + /// Gets a value indicating whether to include the $schema keyword in created schemas. /// public bool IncludeSchemaKeyword { get; init; } /// /// Gets a value indicating whether to mark all properties as required in the schema. /// + [Obsolete("This property has been deprecated. Use the equivalent property in TransformOptions instead.")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] public bool RequireAllProperties { get; init; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformOptions.cs index c7a035cbbed..46e7476afcf 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformOptions.cs @@ -38,6 +38,11 @@ public sealed record class AIJsonSchemaTransformOptions /// public bool UseNullableKeyword { get; init; } + /// + /// Gets a value indicating whether to move the default keyword to the description field in the schema. + /// + public bool MoveDefaultKeywordToDescription { get; init; } + /// /// Gets the default options instance. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs index 8ab0152c941..33531661813 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs @@ -116,4 +116,11 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(AIFunctionArguments))] [EditorBrowsable(EditorBrowsableState.Never)] // Never use JsonContext directly, use DefaultOptions instead. private sealed partial class JsonContext : JsonSerializerContext; + + [JsonSourceGenerationOptions(JsonSerializerDefaults.Web, + UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false)] + [JsonSerializable(typeof(JsonNode))] + private sealed partial class JsonContextNoIndentation : JsonSerializerContext; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Create.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Create.cs index fe17a2ad449..a44836d8e96 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Create.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Create.cs @@ -5,7 +5,6 @@ using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Text.Json; @@ -40,7 +39,7 @@ public static partial class AIJsonUtilities private const string DefaultPropertyName = "default"; private const string RefPropertyName = "$ref"; - /// The uri used when populating the $schema keyword in inferred schemas. + /// The uri used when populating the $schema keyword in created schemas. private const string SchemaKeywordUri = "https://json-schema.org/draft/2020-12/schema"; // List of keywords used by JsonSchemaExporter but explicitly disallowed by some AI vendors. @@ -54,7 +53,7 @@ public static partial class AIJsonUtilities /// The title keyword used by the method schema. /// The description keyword used by the method schema. /// The options used to extract the schema from the specified type. - /// The options controlling schema inference. + /// The options controlling schema creation. /// A JSON schema document encoded as a . /// is . public static JsonElement CreateFunctionJsonSchema( @@ -106,13 +105,13 @@ public static JsonElement CreateFunctionJsonSchema( inferenceOptions); parameterSchemas.Add(parameter.Name, parameterSchema); - if (!parameter.IsOptional || inferenceOptions.RequireAllProperties) + if (!parameter.IsOptional) { (requiredProperties ??= []).Add((JsonNode)parameter.Name); } } - JsonObject schema = new(); + JsonNode schema = new JsonObject(); if (inferenceOptions.IncludeSchemaKeyword) { schema[SchemaPropertyName] = SchemaKeywordUri; @@ -136,7 +135,13 @@ public static JsonElement CreateFunctionJsonSchema( schema[RequiredPropertyName] = requiredProperties; } - return JsonSerializer.SerializeToElement(schema, JsonContext.Default.JsonNode); + // Finally, apply any schema transformations if specified. + if (inferenceOptions.TransformOptions is { } options) + { + schema = TransformSchema(schema, options); + } + + return JsonSerializer.SerializeToElement(schema, JsonContextNoIndentation.Default.JsonNode); } /// Creates a JSON schema for the specified type. @@ -145,7 +150,7 @@ public static JsonElement CreateFunctionJsonSchema( /// if the parameter is optional; otherwise, . /// The default value of the optional parameter, if applicable. /// The options used to extract the schema from the specified type. - /// The options controlling schema inference. + /// The options controlling schema creation. /// A representing the schema. public static JsonElement CreateJsonSchema( Type? type, @@ -158,7 +163,14 @@ public static JsonElement CreateJsonSchema( serializerOptions ??= DefaultOptions; inferenceOptions ??= AIJsonSchemaCreateOptions.Default; JsonNode schema = CreateJsonSchemaCore(type, parameterName: null, description, hasDefaultValue, defaultValue, serializerOptions, inferenceOptions); - return JsonSerializer.SerializeToElement(schema, JsonContext.Default.JsonNode); + + // Finally, apply any schema transformations if specified. + if (inferenceOptions.TransformOptions is { } options) + { + schema = TransformSchema(schema, options); + } + + return JsonSerializer.SerializeToElement(schema, JsonContextNoIndentation.Default.JsonNode); } /// Gets the default JSON schema to be used by types or functions. @@ -203,25 +215,11 @@ private static JsonNode CreateJsonSchemaCore( if (hasDefaultValue) { - if (inferenceOptions.RequireAllProperties) - { - // Default values are only used in the context of optional parameters. - // Do not include a default keyword (since certain AI vendors don't support it) - // and instead embed its JSON in the description as a hint to the LLM. - string defaultValueJson = defaultValue is not null - ? JsonSerializer.Serialize(defaultValue, serializerOptions.GetTypeInfo(defaultValue.GetType())) - : "null"; - - description = CreateDescriptionWithDefaultValue(description, defaultValueJson); - } - else - { - JsonNode? defaultValueNode = defaultValue is not null - ? JsonSerializer.SerializeToNode(defaultValue, serializerOptions.GetTypeInfo(defaultValue.GetType())) - : null; + JsonNode? defaultValueNode = defaultValue is not null + ? JsonSerializer.SerializeToNode(defaultValue, serializerOptions.GetTypeInfo(defaultValue.GetType())) + : null; - (schemaObj ??= [])[DefaultPropertyName] = defaultValueNode; - } + (schemaObj ??= [])[DefaultPropertyName] = defaultValueNode; } if (description is not null) @@ -271,41 +269,11 @@ JsonNode TransformSchemaNode(JsonSchemaExporterContext schemaExporterContext, Js } // Include the type keyword in enum types - if (inferenceOptions.IncludeTypeInEnumSchemas && ctx.TypeInfo.Type.IsEnum && objSchema.ContainsKey(EnumPropertyName) && !objSchema.ContainsKey(TypePropertyName)) + if (ctx.TypeInfo.Type.IsEnum && objSchema.ContainsKey(EnumPropertyName) && !objSchema.ContainsKey(TypePropertyName)) { objSchema.InsertAtStart(TypePropertyName, "string"); } - // Disallow additional properties in object schemas - if (inferenceOptions.DisallowAdditionalProperties && - objSchema.ContainsKey(PropertiesPropertyName) && - !objSchema.ContainsKey(AdditionalPropertiesPropertyName)) - { - objSchema.Add(AdditionalPropertiesPropertyName, (JsonNode)false); - } - - // Mark all properties as required - if (inferenceOptions.RequireAllProperties && - objSchema.TryGetPropertyValue(PropertiesPropertyName, out JsonNode? properties) && - properties is JsonObject propertiesObj) - { - _ = objSchema.TryGetPropertyValue(RequiredPropertyName, out JsonNode? required); - if (required is not JsonArray { } requiredArray || requiredArray.Count != propertiesObj.Count) - { - requiredArray = [.. propertiesObj.Select(prop => (JsonNode)prop.Key)]; - objSchema[RequiredPropertyName] = requiredArray; - } - } - - // Strip default keywords and embed in description where required - if (inferenceOptions.RequireAllProperties && - objSchema.TryGetPropertyValue(DefaultPropertyName, out JsonNode? defaultValue)) - { - _ = objSchema.Remove(DefaultPropertyName); - string defaultValueJson = defaultValue?.ToJsonString() ?? "null"; - localDescription = CreateDescriptionWithDefaultValue(localDescription, defaultValueJson); - } - // Filter potentially disallowed keywords. foreach (string keyword in _schemaKeywordsDisallowedByAIVendors) { @@ -328,20 +296,8 @@ JsonNode TransformSchemaNode(JsonSchemaExporterContext schemaExporterContext, Js if (ctx.Path.IsEmpty && hasDefaultValue) { - // Add root-level default value metadata - if (inferenceOptions.RequireAllProperties) - { - // Default values are only used in the context of optional parameters. - // Do not include a default keyword (since certain AI vendors don't support it) - // and instead embed its JSON in the description as a hint to the LLM. - string defaultValueJson = JsonSerializer.Serialize(defaultValue, ctx.TypeInfo); - localDescription = CreateDescriptionWithDefaultValue(localDescription, defaultValueJson); - } - else - { - JsonNode? defaultValueNode = JsonSerializer.SerializeToNode(defaultValue, ctx.TypeInfo); - ConvertSchemaToObject(ref schema)[DefaultPropertyName] = defaultValueNode; - } + JsonNode? defaultValueNode = JsonSerializer.SerializeToNode(defaultValue, ctx.TypeInfo); + ConvertSchemaToObject(ref schema)[DefaultPropertyName] = defaultValueNode; } if (localDescription is not null) @@ -423,7 +379,7 @@ private static void InsertAtStart(this JsonObject jsonObject, string key, JsonNo jsonObject.Insert(0, key, value); #else jsonObject.Remove(key); - var copiedEntries = jsonObject.ToArray(); + var copiedEntries = System.Linq.Enumerable.ToArray(jsonObject); jsonObject.Clear(); jsonObject.Add(key, value); @@ -434,13 +390,6 @@ private static void InsertAtStart(this JsonObject jsonObject, string key, JsonNo #endif } - private static string CreateDescriptionWithDefaultValue(string? existingDescription, string defaultValueJson) - { - return existingDescription is null - ? $"Default value: {defaultValueJson}" - : $"{existingDescription} (Default value: {defaultValueJson})"; - } - private static JsonElement ParseJsonElement(ReadOnlySpan utf8Json) { Utf8JsonReader reader = new(utf8Json); diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Transform.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Transform.cs index 5669a3fb264..865b4543abb 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Transform.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Transform.cs @@ -30,9 +30,14 @@ public static JsonElement TransformSchema(JsonElement schema, AIJsonSchemaTransf } JsonNode? nodeSchema = JsonSerializer.SerializeToNode(schema, JsonContext.Default.JsonElement); + JsonNode transformedSchema = TransformSchema(nodeSchema, transformOptions); + return JsonSerializer.SerializeToElement(transformedSchema, JsonContextNoIndentation.Default.JsonNode); + } + + private static JsonNode TransformSchema(JsonNode? schema, AIJsonSchemaTransformOptions transformOptions) + { List? path = transformOptions.TransformSchemaNode is not null ? [] : null; - JsonNode transformedSchema = TransformSchemaCore(nodeSchema, transformOptions, path); - return JsonSerializer.Deserialize(transformedSchema, JsonContext.Default.JsonElement); + return TransformSchemaCore(schema, transformOptions, path); } private static JsonNode TransformSchemaCore(JsonNode? schema, AIJsonSchemaTransformOptions transformOptions, List? path) @@ -169,6 +174,18 @@ private static JsonNode TransformSchemaCore(JsonNode? schema, AIJsonSchemaTransf } } + if (transformOptions.MoveDefaultKeywordToDescription && + schemaObj.TryGetPropertyValue(DefaultPropertyName, out JsonNode? defaultSchema)) + { + string? description = schemaObj.TryGetPropertyValue(DescriptionPropertyName, out JsonNode? descriptionSchema) ? descriptionSchema?.GetValue() : null; + string defaultValueJson = JsonSerializer.Serialize(defaultSchema, JsonContextNoIndentation.Default.JsonNode!); + description = description is null + ? $"Default value: {defaultValueJson}" + : $"{description} (Default value: {defaultValueJson})"; + schemaObj[DescriptionPropertyName] = description; + _ = schemaObj.Remove(DefaultPropertyName); + } + break; default: diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs index ff62845eb0e..6c0acb8ee23 100644 --- a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs @@ -29,7 +29,8 @@ internal sealed class AzureAIInferenceChatClient : IChatClient { RequireAllProperties = true, DisallowAdditionalProperties = true, - ConvertBooleanSchemas = true + ConvertBooleanSchemas = true, + MoveDefaultKeywordToDescription = true, }); /// Metadata about the client. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index 98cf49fd696..001f4d1a593 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -30,7 +30,8 @@ internal sealed partial class OpenAIChatClient : IChatClient { RequireAllProperties = true, DisallowAdditionalProperties = true, - ConvertBooleanSchemas = true + ConvertBooleanSchemas = true, + MoveDefaultKeywordToDescription = true, }); /// Gets the default OpenAI endpoint. diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs index e35f8b87949..69c4cc7ee89 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs @@ -26,9 +26,12 @@ public static partial class ChatClientStructuredOutputExtensions private static readonly AIJsonSchemaCreateOptions _inferenceOptions = new() { IncludeSchemaKeyword = true, - DisallowAdditionalProperties = true, - IncludeTypeInEnumSchemas = true, - RequireAllProperties = true, + TransformOptions = new AIJsonSchemaTransformOptions + { + DisallowAdditionalProperties = true, + RequireAllProperties = true, + MoveDefaultKeywordToDescription = true, + }, }; /// Sends chat messages, requesting a response matching the type . diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs index 2d1967b11c1..0001b8b2125 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs @@ -15,6 +15,8 @@ using Microsoft.Extensions.AI.JsonSchemaExporter; using Xunit; +#pragma warning disable 0618 // Suppress obsolete warnings + namespace Microsoft.Extensions.AI; public static partial class AIJsonUtilitiesTests @@ -72,10 +74,11 @@ public static void AIJsonSchemaCreateOptions_DefaultInstance_ReturnsExpectedValu { AIJsonSchemaCreateOptions options = useSingleton ? AIJsonSchemaCreateOptions.Default : new AIJsonSchemaCreateOptions(); Assert.True(options.IncludeTypeInEnumSchemas); - Assert.True(options.DisallowAdditionalProperties); + Assert.False(options.DisallowAdditionalProperties); Assert.False(options.IncludeSchemaKeyword); Assert.False(options.RequireAllProperties); Assert.Null(options.TransformSchemaNode); + Assert.Null(options.TransformOptions); } [Fact] @@ -106,6 +109,12 @@ public static void AIJsonSchemaCreateOptions_UsesStructuralEquality() property.SetValue(options2, includeParameter); break; + case null when property.PropertyType == typeof(AIJsonSchemaTransformOptions): + AIJsonSchemaTransformOptions transformOptions = new AIJsonSchemaTransformOptions { RequireAllProperties = true }; + property.SetValue(options1, transformOptions); + property.SetValue(options2, transformOptions); + break; + default: Assert.Fail($"Unexpected property type: {property.PropertyType}"); break; @@ -152,8 +161,7 @@ public static void CreateJsonSchema_DefaultParameters_GeneratesExpectedJsonSchem "default": "defaultValue" } }, - "required": ["Key", "EnumValue"], - "additionalProperties": false + "required": ["Key", "EnumValue"] } """).RootElement; @@ -176,6 +184,7 @@ public static void CreateJsonSchema_OverriddenParameters_GeneratesExpectedJsonSc "type": "integer" }, "EnumValue": { + "type": "string", "enum": ["A", "B"] }, "Value": { @@ -183,16 +192,20 @@ public static void CreateJsonSchema_OverriddenParameters_GeneratesExpectedJsonSc "type": ["string", "null"] } }, - "required": ["Key", "EnumValue", "Value"] + "required": ["Key", "EnumValue", "Value"], + "additionalProperties": false } """).RootElement; AIJsonSchemaCreateOptions inferenceOptions = new AIJsonSchemaCreateOptions { - IncludeTypeInEnumSchemas = false, - DisallowAdditionalProperties = false, IncludeSchemaKeyword = true, - RequireAllProperties = true, + TransformOptions = new() + { + DisallowAdditionalProperties = true, + RequireAllProperties = true, + MoveDefaultKeywordToDescription = true, + } }; JsonElement actual = AIJsonUtilities.CreateJsonSchema( @@ -227,8 +240,7 @@ public static void CreateJsonSchema_UserDefinedTransformer() "default": "defaultValue" } }, - "required": ["Key", "EnumValue"], - "additionalProperties": false + "required": ["Key", "EnumValue"] } """).RootElement; @@ -268,8 +280,7 @@ public static void CreateJsonSchema_FiltersDisallowedKeywords() "Char" : { "type": "string" } - }, - "additionalProperties": false + } } """).RootElement; @@ -341,6 +352,15 @@ public static void CreateFunctionJsonSchema_OptionalParameters(bool requireAllPr } """).RootElement; + AIJsonSchemaCreateOptions inferenceOptions = new() + { + TransformOptions = new() + { + RequireAllProperties = requireAllProperties, + MoveDefaultKeywordToDescription = requireAllProperties, + } + }; + AIFunction func = AIFunctionFactory.Create(( [Description("The city to get the weather for")] string city, [Description("The unit to calculate the current temperature to")] string unit = "celsius") => "sunny", @@ -348,7 +368,7 @@ public static void CreateFunctionJsonSchema_OptionalParameters(bool requireAllPr { Name = "get_weather", Description = "Gets the current weather for a current location", - JsonSchemaCreateOptions = new AIJsonSchemaCreateOptions { RequireAllProperties = requireAllProperties } + JsonSchemaCreateOptions = inferenceOptions }); Assert.NotNull(func.UnderlyingMethod); @@ -358,7 +378,7 @@ public static void CreateFunctionJsonSchema_OptionalParameters(bool requireAllPr func.UnderlyingMethod, title: func.Name, description: func.Description, - inferenceOptions: new AIJsonSchemaCreateOptions { RequireAllProperties = requireAllProperties }); + inferenceOptions: inferenceOptions); AssertDeepEquals(expected, resolvedSchema); } @@ -423,7 +443,7 @@ public static void CreateJsonSchema_ValidateWithTestData(ITestData testData) JsonTypeInfo typeInfo = options.GetTypeInfo(testData.Type); AIJsonSchemaCreateOptions? createOptions = typeInfo.Properties.Any(prop => prop.IsExtensionData) - ? new() { DisallowAdditionalProperties = false } // Do not append additionalProperties: false to the schema if the type has extension data. + ? new() { TransformOptions = new() { DisallowAdditionalProperties = false } } // Do not append additionalProperties: false to the schema if the type has extension data. : null; JsonElement schema = AIJsonUtilities.CreateJsonSchema(testData.Type, serializerOptions: options, inferenceOptions: createOptions); @@ -706,6 +726,33 @@ public static void TransformJsonSchema_UseNullableKeyword() AssertDeepEquals(expectedSchema, transformedSchema); } + [Fact] + public static void TransformJsonSchema_MoveDefaultKeywordToDescription() + { + JsonElement schema = JsonDocument.Parse(""" + { + "description": "My awesome schema", + "type": "array", + "default": [1,2,3] + } + """).RootElement; + + JsonElement expectedSchema = JsonDocument.Parse(""" + { + "description": "My awesome schema (Default value: [1,2,3])", + "type": "array" + } + """).RootElement; + + AIJsonSchemaTransformOptions options = new() + { + MoveDefaultKeywordToDescription = true, + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, options); + AssertDeepEquals(expectedSchema, transformedSchema); + } + [Theory] [MemberData(nameof(TestTypes.GetTestDataUsingAllValues), MemberType = typeof(TestTypes))] public static void TransformJsonSchema_ValidateWithTestData(ITestData testData) @@ -718,7 +765,7 @@ public static void TransformJsonSchema_ValidateWithTestData(ITestData testData) JsonTypeInfo typeInfo = options.GetTypeInfo(testData.Type); AIJsonSchemaCreateOptions? createOptions = typeInfo.Properties.Any(prop => prop.IsExtensionData) - ? new() { DisallowAdditionalProperties = false } // Do not append additionalProperties: false to the schema if the type has extension data. + ? new() { TransformOptions = new() { DisallowAdditionalProperties = false } } // Do not append additionalProperties: false to the schema if the type has extension data. : null; JsonElement schema = AIJsonUtilities.CreateJsonSchema(testData.Type, serializerOptions: options, inferenceOptions: createOptions); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs index aae985c4c4b..edd22edc41e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs @@ -34,7 +34,8 @@ public async Task SuccessUsage_Default() GetResponseAsyncCallback = (messages, options, cancellationToken) => { var responseFormat = Assert.IsType(options!.ResponseFormat); - Assert.Equal(""" + Assert.NotNull(responseFormat.Schema); + AssertDeepEquals(JsonDocument.Parse(""" { "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Some test description", @@ -65,7 +66,7 @@ public async Task SuccessUsage_Default() "species" ] } - """, responseFormat.Schema.ToString()); + """).RootElement, responseFormat.Schema.Value); Assert.Equal(nameof(Animal), responseFormat.SchemaName); Assert.Equal("Some test description", responseFormat.SchemaDescription); @@ -332,7 +333,8 @@ public async Task CanSpecifyCustomJsonSerializationOptions() // - The property is named full_name, because we specified SnakeCaseLower // - The species value is an integer instead of a string, because we didn't use enum-to-string conversion var responseFormat = Assert.IsType(options!.ResponseFormat); - Assert.Equal(""" + Assert.NotNull(responseFormat.Schema); + AssertDeepEquals(JsonDocument.Parse(""" { "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Some test description", @@ -358,7 +360,7 @@ public async Task CanSpecifyCustomJsonSerializationOptions() "species" ] } - """, responseFormat.Schema.ToString()); + """).RootElement, responseFormat.Schema.Value); return Task.FromResult(expectedResponse); }, @@ -432,4 +434,28 @@ private class Envelope [JsonSerializable(typeof(Envelope))] [JsonSerializable(typeof(Data))] private partial class JsonContext2 : JsonSerializerContext; + + private static void AssertDeepEquals(JsonElement element1, JsonElement element2) + { +#pragma warning disable SA1118 // Parameter should not span multiple lines + Assert.True(DeepEquals(element1, element2), $""" + Elements are not equal. + Expected: + {element1} + Actual: + {element2} + """); +#pragma warning restore SA1118 // Parameter should not span multiple lines + } + + private static bool DeepEquals(JsonElement element1, JsonElement element2) + { +#if NET9_0_OR_GREATER + return JsonElement.DeepEquals(element1, element2); +#else + return System.Text.Json.Nodes.JsonNode.DeepEquals( + JsonSerializer.SerializeToNode(element1, AIJsonUtilities.DefaultOptions), + JsonSerializer.SerializeToNode(element2, AIJsonUtilities.DefaultOptions)); +#endif + } } From c2f7b3867ccd03a54a89e4a08af7b8ccfc9ff913 Mon Sep 17 00:00:00 2001 From: Jeff Handley Date: Wed, 14 May 2025 00:19:48 -0700 Subject: [PATCH 25/26] Update MEAI Template test snapshots --- .../aichatweb.Basic.verified/aichatweb/aichatweb.csproj | 2 +- .../aichatweb/aichatweb.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj index 4b1fad034a9..78946f3f313 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj @@ -9,7 +9,7 @@ - + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj index fd7131e492a..09532951c55 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj @@ -10,7 +10,7 @@ - + From 5aab00ef6abd50313bc5150a6699150cd2d2927d Mon Sep 17 00:00:00 2001 From: Jeff Handley Date: Wed, 14 May 2025 00:47:42 -0700 Subject: [PATCH 26/26] Pin the non-AI package versions for the MEAI Templates --- src/ProjectTemplates/GeneratedContent.targets | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ProjectTemplates/GeneratedContent.targets b/src/ProjectTemplates/GeneratedContent.targets index 832d533e66a..5842d5cad74 100644 --- a/src/ProjectTemplates/GeneratedContent.targets +++ b/src/ProjectTemplates/GeneratedContent.targets @@ -15,12 +15,12 @@ - 9.4.0 - 9.4.0-preview.1.25207.5 - 9.0.4 + 9.5.0 + 9.5.0-preview.1.25262.9 + 9.0.5 - false + true false