diff --git a/src/Infrastructure/BotSharp.Abstraction/Entity/IEntityAnalyzer.cs b/src/Infrastructure/BotSharp.Abstraction/Entity/IEntityAnalyzer.cs new file mode 100644 index 000000000..ca6a8680c --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Entity/IEntityAnalyzer.cs @@ -0,0 +1,11 @@ +using BotSharp.Abstraction.Entity.Models; +using BotSharp.Abstraction.Entity.Responses; + +namespace BotSharp.Abstraction.Entity; + +public interface IEntityAnalyzer +{ + string Provider { get; } + + Task AnalyzeAsync(string text, EntityAnalysisOptions? options = null); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Tokenizers/ITokenDataLoader.cs b/src/Infrastructure/BotSharp.Abstraction/Entity/IEntityDataLoader.cs similarity index 86% rename from src/Infrastructure/BotSharp.Abstraction/Tokenizers/ITokenDataLoader.cs rename to src/Infrastructure/BotSharp.Abstraction/Entity/IEntityDataLoader.cs index 745dfd914..caf8fe227 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Tokenizers/ITokenDataLoader.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Entity/IEntityDataLoader.cs @@ -1,6 +1,6 @@ -namespace BotSharp.Abstraction.Tokenizers; +namespace BotSharp.Abstraction.Entity; -public interface ITokenDataLoader +public interface IEntityDataLoader { string Provider { get; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Tokenizers/Models/TokenizeOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs similarity index 91% rename from src/Infrastructure/BotSharp.Abstraction/Tokenizers/Models/TokenizeOptions.cs rename to src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs index 1f1c70f25..b0e08dc63 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Tokenizers/Models/TokenizeOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs @@ -1,6 +1,6 @@ -namespace BotSharp.Abstraction.Tokenizers.Models; +namespace BotSharp.Abstraction.Entity.Models; -public class TokenizeOptions +public class EntityAnalysisOptions { /// /// Token data providers diff --git a/src/Infrastructure/BotSharp.Abstraction/Tokenizers/Models/TokenizeResult.cs b/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisResult.cs similarity index 77% rename from src/Infrastructure/BotSharp.Abstraction/Tokenizers/Models/TokenizeResult.cs rename to src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisResult.cs index a02a716d2..c760456ca 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Tokenizers/Models/TokenizeResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisResult.cs @@ -1,6 +1,6 @@ -namespace BotSharp.Abstraction.Tokenizers.Models; +namespace BotSharp.Abstraction.Entity.Models; -public class TokenizeResult +public class EntityAnalysisResult { public string Token { get; set; } = string.Empty; diff --git a/src/Infrastructure/BotSharp.Abstraction/Entity/Responses/EntityAnalysisResponse.cs b/src/Infrastructure/BotSharp.Abstraction/Entity/Responses/EntityAnalysisResponse.cs new file mode 100644 index 000000000..143e24f64 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Entity/Responses/EntityAnalysisResponse.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.Entity.Models; + +namespace BotSharp.Abstraction.Entity.Responses; + +public class EntityAnalysisResponse : ResponseBase +{ + public List Results { get; set; } = []; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Tokenizers/ITokenizer.cs b/src/Infrastructure/BotSharp.Abstraction/Tokenizers/ITokenizer.cs deleted file mode 100644 index b13663899..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Tokenizers/ITokenizer.cs +++ /dev/null @@ -1,11 +0,0 @@ -using BotSharp.Abstraction.Tokenizers.Models; -using BotSharp.Abstraction.Tokenizers.Responses; - -namespace BotSharp.Abstraction.Tokenizers; - -public interface ITokenizer -{ - string Provider { get; } - - Task TokenizeAsync(string text, TokenizeOptions? options = null); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Tokenizers/Responses/TokenizeResponse.cs b/src/Infrastructure/BotSharp.Abstraction/Tokenizers/Responses/TokenizeResponse.cs deleted file mode 100644 index e01fe153f..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Tokenizers/Responses/TokenizeResponse.cs +++ /dev/null @@ -1,8 +0,0 @@ -using BotSharp.Abstraction.Tokenizers.Models; - -namespace BotSharp.Abstraction.Tokenizers.Responses; - -public class TokenizeResponse : ResponseBase -{ - public List Results { get; set; } = []; -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs index 7411e98b9..cdd91c2dd 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Document.cs @@ -9,8 +9,8 @@ namespace BotSharp.OpenAPI.Controllers; public partial class KnowledgeBaseController { #region Document - [HttpGet("/knowledge/document/processors")] - public IEnumerable GetKnowledgeDocumentProcessors() + [HttpGet("/knowledge/processors")] + public IEnumerable GetKnowledgeProcessors() { return _services.GetServices().Select(x => x.Provider); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs new file mode 100644 index 000000000..266155539 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Entity.cs @@ -0,0 +1,46 @@ +using BotSharp.OpenAPI.ViewModels.Knowledges; + +namespace BotSharp.OpenAPI.Controllers; + +public partial class KnowledgeBaseController +{ + /// + /// Entity analyis with options + /// + /// + /// + [HttpPost("knowledge/entity/analyze")] + public async Task EntityAnalyze([FromBody] EntityAnalysisRequest request) + { + var analyzer = _services.GetServices() + .FirstOrDefault(x => x.Provider.IsEqualTo(request.Provider)); + + if (analyzer == null) + { + return null; + } + return await analyzer.AnalyzeAsync(request.Text, request.Options); + } + + /// + /// Get entity analyzers + /// + /// + [HttpGet("knowledge/entity/analyzers")] + public IEnumerable GetEntityAnalyzers() + { + var analyzers = _services.GetServices(); + return analyzers.Select(x => x.Provider); + } + + /// + /// Get entity data providers + /// + /// + [HttpGet("knowledge/entity/data-providers")] + public IEnumerable GetEntityDataProviders() + { + var dataLoaders = _services.GetServices(); + return dataLoaders.Select(x => x.Provider); + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Tokenizer.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Tokenizer.cs deleted file mode 100644 index 08f7812fc..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBase/KnowledgeBaseController.Tokenizer.cs +++ /dev/null @@ -1,48 +0,0 @@ -using BotSharp.Abstraction.Tokenizers; -using BotSharp.Abstraction.Tokenizers.Responses; -using BotSharp.OpenAPI.ViewModels.Knowledges; - -namespace BotSharp.OpenAPI.Controllers; - -public partial class KnowledgeBaseController -{ - /// - /// Tokenize text with options - /// - /// - /// - [HttpPost("knowledge/tokenize")] - public async Task Tokenize([FromBody] TokenizeRequest request) - { - var tokenizer = _services.GetServices() - .FirstOrDefault(x => x.Provider.IsEqualTo(request.Provider)); - - if (tokenizer == null) - { - return null; - } - return await tokenizer.TokenizeAsync(request.Text, request.Options); - } - - /// - /// Get tokenizer providers - /// - /// - [HttpGet("knowledge/tokenizer/providers")] - public IEnumerable GetTokenizerProviders() - { - var tokenizers = _services.GetServices(); - return tokenizers.Select(x => x.Provider); - } - - /// - /// Get token data loader providers - /// - /// - [HttpGet("knowledge/tokenizer/data-providers")] - public IEnumerable GetTokenizerDataProviders() - { - var dataLoaders = _services.GetServices(); - return dataLoaders.Select(x => x.Provider); - } -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/Using.cs b/src/Infrastructure/BotSharp.OpenAPI/Using.cs index ea5f9f7cc..d811452ee 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Using.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Using.cs @@ -29,6 +29,10 @@ global using BotSharp.Abstraction.VectorStorage.Enums; global using BotSharp.Abstraction.Knowledges.Models; global using BotSharp.Abstraction.Chart.Models; +global using BotSharp.Abstraction.Entity; +global using BotSharp.Abstraction.Entity.Models; +global using BotSharp.Abstraction.Entity.Responses; + global using BotSharp.OpenAPI.ViewModels.Conversations; global using BotSharp.OpenAPI.ViewModels.Users; global using BotSharp.OpenAPI.ViewModels.Agents; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/TokenizeRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/EntityAnalysisRequest.cs similarity index 54% rename from src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/TokenizeRequest.cs rename to src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/EntityAnalysisRequest.cs index 125013d0d..890113448 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/TokenizeRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/Request/EntityAnalysisRequest.cs @@ -1,10 +1,8 @@ -using BotSharp.Abstraction.Tokenizers.Models; - namespace BotSharp.OpenAPI.ViewModels.Knowledges; -public class TokenizeRequest +public class EntityAnalysisRequest { public string Text { get; set; } = string.Empty; public string? Provider { get; set; } - public TokenizeOptions? Options { get; set; } + public EntityAnalysisOptions? Options { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs b/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs index fc1a75326..cc6f3e8f1 100644 --- a/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs @@ -19,8 +19,8 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/DataLoaders/CsvTokenDataLoader.cs b/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/DataLoaders/CsvNERDataLoader.cs similarity index 97% rename from src/Plugins/BotSharp.Plugin.FuzzySharp/Services/DataLoaders/CsvTokenDataLoader.cs rename to src/Plugins/BotSharp.Plugin.FuzzySharp/Services/DataLoaders/CsvNERDataLoader.cs index beca5c6b3..1e1c2b949 100644 --- a/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/DataLoaders/CsvTokenDataLoader.cs +++ b/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/DataLoaders/CsvNERDataLoader.cs @@ -8,14 +8,14 @@ namespace BotSharp.Plugin.FuzzySharp.Services.DataLoaders; -public class CsvTokenDataLoader : ITokenDataLoader +public class CsvNERDataLoader : IEntityDataLoader { - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly FuzzySharpSettings _settings; private readonly string _basePath; - public CsvTokenDataLoader( - ILogger logger, + public CsvNERDataLoader( + ILogger logger, FuzzySharpSettings settings) { _settings = settings; diff --git a/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/FuzzySharpTokenizer.cs b/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/FuzzySharpEntityAnalyzer.cs similarity index 90% rename from src/Plugins/BotSharp.Plugin.FuzzySharp/Services/FuzzySharpTokenizer.cs rename to src/Plugins/BotSharp.Plugin.FuzzySharp/Services/FuzzySharpEntityAnalyzer.cs index 863cda1b0..f17afafe4 100644 --- a/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/FuzzySharpTokenizer.cs +++ b/src/Plugins/BotSharp.Plugin.FuzzySharp/Services/FuzzySharpEntityAnalyzer.cs @@ -3,16 +3,16 @@ namespace BotSharp.Plugin.FuzzySharp.Services; -public class FuzzySharpTokenizer : ITokenizer +public class FuzzySharpEntityAnalyzer : IEntityAnalyzer { - private readonly ILogger _logger; - private readonly IEnumerable _tokenDataLoaders; + private readonly ILogger _logger; + private readonly IEnumerable _tokenDataLoaders; private readonly INgramProcessor _ngramProcessor; private readonly IResultProcessor _resultProcessor; - public FuzzySharpTokenizer( - ILogger logger, - IEnumerable tokenDataLoaders, + public FuzzySharpEntityAnalyzer( + ILogger logger, + IEnumerable tokenDataLoaders, INgramProcessor ngramProcessor, IResultProcessor resultProcessor) { @@ -24,18 +24,17 @@ public FuzzySharpTokenizer( public string Provider => "fuzzy-sharp"; - public async Task TokenizeAsync(string text, TokenizeOptions? options = null) + public async Task AnalyzeAsync(string text, EntityAnalysisOptions? options = null) { - var response = new TokenizeResponse(); + var response = new EntityAnalysisResponse(); try { var result = await AnalyzeTextAsync(text, options); - - return new TokenizeResponse + return new EntityAnalysisResponse { Success = true, - Results = result?.FlaggedItems?.Select(f => new TokenizeResult + Results = result?.FlaggedItems?.Select(f => new EntityAnalysisResult { Token = f.Token, CanonicalText = f.CanonicalForm, @@ -59,7 +58,7 @@ public async Task TokenizeAsync(string text, TokenizeOptions? /// /// Analyze text for typos and entities using domain-specific vocabulary /// - private async Task AnalyzeTextAsync(string text, TokenizeOptions? options = null) + private async Task AnalyzeTextAsync(string text, EntityAnalysisOptions? options = null) { var stopwatch = Stopwatch.StartNew(); try @@ -148,7 +147,7 @@ private List AnalyzeTokens( List tokens, Dictionary> vocabulary, Dictionary synonymMapping, - TokenizeOptions? options) + EntityAnalysisOptions? options) { // Build lookup table for O(1) exact match lookups (matching Python's build_lookup) var lookup = BuildLookup(vocabulary); diff --git a/src/Plugins/BotSharp.Plugin.FuzzySharp/Using.cs b/src/Plugins/BotSharp.Plugin.FuzzySharp/Using.cs index 9a6ec03d5..776965b5e 100644 --- a/src/Plugins/BotSharp.Plugin.FuzzySharp/Using.cs +++ b/src/Plugins/BotSharp.Plugin.FuzzySharp/Using.cs @@ -4,9 +4,9 @@ global using System.Text; global using System.Threading.Tasks; -global using BotSharp.Abstraction.Tokenizers; -global using BotSharp.Abstraction.Tokenizers.Models; -global using BotSharp.Abstraction.Tokenizers.Responses; +global using BotSharp.Abstraction.Entity; +global using BotSharp.Abstraction.Entity.Models; +global using BotSharp.Abstraction.Entity.Responses; global using BotSharp.Plugin.FuzzySharp.Models; global using BotSharp.Plugin.FuzzySharp.Utils;