Merge Elastic.Clients.AgentBuilder and AgentBuilder MEAI extensions

Directory.Packages.props

@@ -6,14 +6,21 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageVersion Include="Elastic.Clients.Elasticsearch" Version="8.19.12" />
+    <PackageVersion Include="Elastic.Transport" Version="0.17.3" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.10" />
     <PackageVersion Include="Microsoft.Extensions.VectorData.Abstractions" Version="9.7.0" />
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
     <PackageVersion Include="MinVer" Version="6.0.0" />
     <PackageVersion Include="PolySharp" Version="1.15.0" />
     <PackageVersion Include="System.Text.Json" Version="9.0.10" />
+    <PackageVersion Include="Microsoft.Bcl.Memory" Version="10.0.5" />
+    <PackageVersion Include="Microsoft.Extensions.AI" Version="10.4.1" />
+    <PackageVersion Include="ModelContextProtocol" Version="1.1.0" />
+    <PackageVersion Include="System.Net.ServerSentEvents" Version="10.0.5" />
     <!-- Tests -->
     <PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.10.2" />
+    <PackageVersion Include="FluentAssertions" Version="7.2.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="9.0.10" />
     <PackageVersion Include="Microsoft.Extensions.Configuration" Version="9.0.10" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="9.0.10" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />

Elastic.SemanticKernel.sln

@@ -25,6 +25,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Extensions.AI", "sr
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Extensions.AI.Tests", "src\Elastic.Extensions.AI.Tests\Elastic.Extensions.AI.Tests.csproj", "{B2C3D4E5-F6A7-8901-BCDE-F12345678901}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Clients.AgentBuilder", "src\Elastic.Clients.AgentBuilder\Elastic.Clients.AgentBuilder.csproj", "{C1D2E3F4-A5B6-7890-CDEF-123456789ABC}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Clients.AgentBuilder.Tests", "test\Elastic.Clients.AgentBuilder.Tests\Elastic.Clients.AgentBuilder.Tests.csproj", "{D2E3F4A5-B6C7-8901-DEFA-234567890BCD}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Clients.AgentBuilder.IntegrationTests", "test\Elastic.Clients.AgentBuilder.IntegrationTests\Elastic.Clients.AgentBuilder.IntegrationTests.csproj", "{E3F4A5B6-C7D8-9012-EFAB-345678901CDE}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elastic.Extensions.AI.IntegrationTests", "test\Elastic.Extensions.AI.IntegrationTests\Elastic.Extensions.AI.IntegrationTests.csproj", "{F4A5B6C7-D8E9-0123-FABC-456789012DEF}"
+EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "MEAI", "MEAI", "{70CFAC87-C106-4574-A748-E6E25ABD1DD1}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "MEVD", "MEVD", "{E7E4A7FF-9524-4C3A-BCFE-A00348D0926F}"
@@ -59,6 +67,22 @@ Global
 		{B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B2C3D4E5-F6A7-8901-BCDE-F12345678901}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C1D2E3F4-A5B6-7890-CDEF-123456789ABC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C1D2E3F4-A5B6-7890-CDEF-123456789ABC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C1D2E3F4-A5B6-7890-CDEF-123456789ABC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C1D2E3F4-A5B6-7890-CDEF-123456789ABC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D2E3F4A5-B6C7-8901-DEFA-234567890BCD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D2E3F4A5-B6C7-8901-DEFA-234567890BCD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D2E3F4A5-B6C7-8901-DEFA-234567890BCD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D2E3F4A5-B6C7-8901-DEFA-234567890BCD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E3F4A5B6-C7D8-9012-EFAB-345678901CDE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E3F4A5B6-C7D8-9012-EFAB-345678901CDE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E3F4A5B6-C7D8-9012-EFAB-345678901CDE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E3F4A5B6-C7D8-9012-EFAB-345678901CDE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F4A5B6C7-D8E9-0123-FABC-456789012DEF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F4A5B6C7-D8E9-0123-FABC-456789012DEF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F4A5B6C7-D8E9-0123-FABC-456789012DEF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F4A5B6C7-D8E9-0123-FABC-456789012DEF}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -70,6 +94,10 @@ Global
 		{BAB86CE5-4E26-2074-A6B6-E560B2C70022} = {E7E4A7FF-9524-4C3A-BCFE-A00348D0926F}
 		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890} = {70CFAC87-C106-4574-A748-E6E25ABD1DD1}
 		{B2C3D4E5-F6A7-8901-BCDE-F12345678901} = {70CFAC87-C106-4574-A748-E6E25ABD1DD1}
+		{C1D2E3F4-A5B6-7890-CDEF-123456789ABC} = {70CFAC87-C106-4574-A748-E6E25ABD1DD1}
+		{D2E3F4A5-B6C7-8901-DEFA-234567890BCD} = {70CFAC87-C106-4574-A748-E6E25ABD1DD1}
+		{E3F4A5B6-C7D8-9012-EFAB-345678901CDE} = {70CFAC87-C106-4574-A748-E6E25ABD1DD1}
+		{F4A5B6C7-D8E9-0123-FABC-456789012DEF} = {70CFAC87-C106-4574-A748-E6E25ABD1DD1}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {DE39841E-28D0-4E3C-9554-55DC9DDA5761}

src/Elastic.Clients.AgentBuilder/AgentBuilderBootstrapper.cs

@@ -0,0 +1,213 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using Elastic.Clients.AgentBuilder.Agents;
+using Elastic.Clients.AgentBuilder.Tools;
+
+namespace Elastic.Clients.AgentBuilder;
+
+/// <summary>
+/// Defines the tools and agents to bootstrap.
+/// </summary>
+public record BootstrapDefinition
+{
+	/// <summary> Tool definitions to create or update. Each must have a unique <c>Id</c>. </summary>
+	public IReadOnlyList<CreateEsqlToolRequest>? EsqlTools { get; init; }
+
+	/// <summary> Index search tool definitions to create or update. </summary>
+	public IReadOnlyList<CreateIndexSearchToolRequest>? IndexSearchTools { get; init; }
+
+	/// <summary> Agent definitions to create or update. Each must have a unique <c>Id</c>. </summary>
+	public IReadOnlyList<CreateAgentRequest>? Agents { get; init; }
+}
+
+/// <summary>
+/// Idempotent bootstrapper for Agent Builder resources.
+/// Computes a hash of each definition and stores it in <c>tags</c> (tools) or <c>labels</c> (agents).
+/// On subsequent runs, resources are only updated if their hash has changed.
+/// </summary>
+public class AgentBuilderBootstrapper
+{
+	private const string HashPrefix = "_hash:";
+
+	private readonly AgentBuilderClient _client;
+
+	public AgentBuilderBootstrapper(AgentBuilderClient client) =>
+		_client = client ?? throw new ArgumentNullException(nameof(client));
+
+	/// <summary>
+	/// Bootstrap all tools and agents defined in <paramref name="definition"/>.
+	/// </summary>
+	public async Task<bool> BootstrapAsync(
+		BootstrapMethod method, BootstrapDefinition definition, CancellationToken ct = default)
+	{
+		if (method == BootstrapMethod.None)
+			return true;
+
+		try
+		{
+			if (definition.EsqlTools != null)
+			{
+				foreach (var tool in definition.EsqlTools)
+					await BootstrapEsqlToolAsync(tool, ct).ConfigureAwait(false);
+			}
+
+			if (definition.IndexSearchTools != null)
+			{
+				foreach (var tool in definition.IndexSearchTools)
+					await BootstrapIndexSearchToolAsync(tool, ct).ConfigureAwait(false);
+			}
+
+			if (definition.Agents != null)
+			{
+				foreach (var agent in definition.Agents)
+					await BootstrapAgentAsync(agent, ct).ConfigureAwait(false);
+			}
+
+			return true;
+		}
+		catch when (method == BootstrapMethod.Silent)
+		{
+			return false;
+		}
+	}
+
+	private async Task BootstrapEsqlToolAsync(CreateEsqlToolRequest request, CancellationToken ct)
+	{
+		var hash = ComputeHash(request, AgentBuilderSerializationContext.Default.CreateEsqlToolRequest);
+		var existing = await TryGetToolAsync(request.Id, ct).ConfigureAwait(false);
+
+		if (existing != null && HasMatchingHash(existing.Tags, hash))
+			return;
+
+		if (existing != null)
+		{
+			await _client.UpdateToolAsync(request.Id, new UpdateEsqlToolRequest
+			{
+				Description = request.Description,
+				Tags = MergeHashTag(request.Tags, hash),
+				Configuration = request.Configuration
+			}, ct).ConfigureAwait(false);
+		}
+		else
+		{
+			var tagged = request with { Tags = MergeHashTag(request.Tags, hash) };
+			await _client.CreateToolAsync(tagged, ct).ConfigureAwait(false);
+		}
+	}
+
+	private async Task BootstrapIndexSearchToolAsync(CreateIndexSearchToolRequest request, CancellationToken ct)
+	{
+		var hash = ComputeHash(request, AgentBuilderSerializationContext.Default.CreateIndexSearchToolRequest);
+		var existing = await TryGetToolAsync(request.Id, ct).ConfigureAwait(false);
+
+		if (existing != null && HasMatchingHash(existing.Tags, hash))
+			return;
+
+		if (existing != null)
+		{
+			await _client.UpdateToolAsync(request.Id, new UpdateIndexSearchToolRequest
+			{
+				Description = request.Description,
+				Tags = MergeHashTag(request.Tags, hash),
+				Configuration = request.Configuration
+			}, ct).ConfigureAwait(false);
+		}
+		else
+		{
+			var tagged = request with { Tags = MergeHashTag(request.Tags, hash) };
+			await _client.CreateToolAsync(tagged, ct).ConfigureAwait(false);
+		}
+	}
+
+	private async Task BootstrapAgentAsync(CreateAgentRequest request, CancellationToken ct)
+	{
+		var hash = ComputeHash(request, AgentBuilderSerializationContext.Default.CreateAgentRequest);
+		AgentBuilderAgent? existing;
+		try
+		{
+			existing = await _client.GetAgentAsync(request.Id, ct).ConfigureAwait(false);
+		}
+		catch (AgentBuilderException ex) when (ex.ApiCallDetails.HttpStatusCode == 404)
+		{
+			existing = null;
+		}
+
+		if (existing != null && HasMatchingHash(existing.Labels, hash))
+			return;
+
+		if (existing != null)
+		{
+			await _client.UpdateAgentAsync(request.Id, new UpdateAgentRequest
+			{
+				Name = request.Name,
+				Description = request.Description,
+				Labels = MergeHashTag(request.Labels, hash),
+				AvatarColor = request.AvatarColor,
+				AvatarSymbol = request.AvatarSymbol,
+				Configuration = request.Configuration
+			}, ct).ConfigureAwait(false);
+		}
+		else
+		{
+			var labeled = request with { Labels = MergeHashTag(request.Labels, hash) };
+			await _client.CreateAgentAsync(labeled, ct).ConfigureAwait(false);
+		}
+	}
+
+	private async Task<AgentBuilderTool?> TryGetToolAsync(string toolId, CancellationToken ct)
+	{
+		try
+		{
+			return await _client.GetToolAsync(toolId, ct).ConfigureAwait(false);
+		}
+		catch (AgentBuilderException ex) when (ex.ApiCallDetails.HttpStatusCode == 404)
+		{
+			return null;
+		}
+	}
+
+	private static bool HasMatchingHash(IReadOnlyList<string>? tagsOrLabels, string hash)
+	{
+		if (tagsOrLabels == null)
+			return false;
+		return tagsOrLabels.Any(t => t == HashPrefix + hash);
+	}
+
+	private static IReadOnlyList<string> MergeHashTag(IReadOnlyList<string>? existing, string hash)
+	{
+		var result = new List<string>();
+		if (existing != null)
+		{
+			foreach (var tag in existing)
+			{
+				if (!tag.StartsWith(HashPrefix, StringComparison.Ordinal))
+					result.Add(tag);
+			}
+		}
+		result.Add(HashPrefix + hash);
+		return result;
+	}
+
+	public static string ComputeHash<T>(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo)
+	{
+		var json = JsonSerializer.Serialize(value, typeInfo);
+		var input = Encoding.UTF8.GetBytes(json);
+#if NET8_0_OR_GREATER
+		var bytes = SHA256.HashData(input);
+#else
+		using var sha = SHA256.Create();
+		var bytes = sha.ComputeHash(input);
+#endif
+		return Convert.ToBase64String(bytes, 0, 12);
+	}
+}

src/Elastic.Clients.AgentBuilder/AgentBuilderClient.Agents.cs

@@ -0,0 +1,38 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Threading;
+using System.Threading.Tasks;
+using Elastic.Clients.AgentBuilder.Agents;
+
+namespace Elastic.Clients.AgentBuilder;
+
+public partial class AgentBuilderClient
+{
+	/// <summary> List all agents. </summary>
+	public Task<ListAgentsResponse> ListAgentsAsync(CancellationToken ct = default) =>
+		GetAsync<ListAgentsResponse>("/agents", ct);
+
+	/// <summary> Get an agent by its ID. </summary>
+	public Task<AgentBuilderAgent> GetAgentAsync(string agentId, CancellationToken ct = default) =>
+		GetAsync<AgentBuilderAgent>($"/agents/{agentId}", ct);
+
+	/// <summary> Create a new agent. </summary>
+	public Task<AgentBuilderAgent> CreateAgentAsync(CreateAgentRequest request, CancellationToken ct = default) =>
+		PostAsync<CreateAgentRequest, AgentBuilderAgent>("/agents", request, ct);
+
+	/// <summary> Update an existing agent. </summary>
+	public Task<AgentBuilderAgent> UpdateAgentAsync(string agentId, UpdateAgentRequest request, CancellationToken ct = default) =>
+		PutAsync<UpdateAgentRequest, AgentBuilderAgent>($"/agents/{agentId}", request, ct);
+
+	/// <summary> Delete an agent by its ID. </summary>
+	public Task DeleteAgentAsync(string agentId, CancellationToken ct = default) =>
+		DeleteAsync($"/agents/{agentId}", ct);
+
+	/// <summary> Get paginated, per-conversation token consumption data for an agent. </summary>
+	public Task<AgentConsumptionResponse> GetAgentConsumptionAsync(
+		string agentId, AgentConsumptionRequest request, CancellationToken ct = default) =>
+		PostAsync<AgentConsumptionRequest, AgentConsumptionResponse>(
+			$"/agents/{agentId}/consumption", request, ct);
+}