.Net: Bug: Auto function throwing "Error: Function call request for a function that wasnt defined" #9850
Comments
markwallace-microsoft
added
.NET
github-actions
bot
changed the title
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;
using OpenTelemetry;
using OpenTelemetry.Logs;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using System.ComponentModel;
class CxObservePlugins
{
[KernelFunction("get_customer_id_by_customer_name")]
[Description("Gets the Customer Id provided the Customer Name. If multiple records are returned, please ask user for further selection")]
[return: Description("The customer id")]
public string get_customer_id_by_customer_name([Description("The customer name")] string customerName)
{
return $"784852"; // Dummy for Testing
}
[KernelFunction("get_workloads_for_customer_with_azores_score")]
[Description("Gets the Workloads and their Azores Score for a workload or customer with additional filtering conditions.")]
[return: Description("The workloads for a customer with their Azores Score")]
public string get_workloads_for_customer_with_azores_score(
[Description("The Workload ID. If this filter is to be used, ALWAYS get the value of workload ID from function get_azores_workload_id_and_name_by_customer_id before proceeding")] string? workloadId = null,
[Description("The Customer ID. If this filter is to be used, ALWAYS get the value of customer ID from function get_customer_id_by_customer_name before proceeding")] string? customerId = null,
[Description("The azores score lower bound threshold. Example: azores score greater than 10 then lower bound is 10")] double? azoresScoreLowerThreshold = null,
[Description("The azores score upper bound threshold. Example: azores score less than 20 then upper bound is 20")] double? azoresScoreUpperThreshold = null,
[Description("The comparison operator for azores score. This can be 'greater than' or 'less than' or 'greater than X and less than Y' or 'less than Y and greater than X', " +
"where X is lower bound and Y is upper bound ")] string? operatorForAzoresThreshold = null,
[Description("The number of resources which are zone resilient")] int? zoneResilientResourceCount = null,
[Description("The comparison operator for zone resilient resource count. This can either be 'greater than' or 'less than'")] string? operatorForZoneResilientResourcesCount = null,
[Description("The total number of resources(sum of zone resilient and non-zone resilient resources)")] int? resourcesCount = null,
[Description("The comparison operator for total resource count. This can either be 'greater than' or 'less than'")] string? operatorForResourcesCount = null,
[Description("The number of records to retrieve. Also known as Top 'N'. Defaults to 10 if not provided")] int topN = 10
)
{
return $"Vibranium (Workload Id: e4afc0ad-f671-428b-97b6-69e8a919b78b); Shared Services - Networking (Workload Id: a08f2a6f-53c2-48d9-8148-c76261a83505); Adamantium(Workload Id: 673ce7cb-5337-41e1-8cb0-5869769c6ac7";
}
[KernelFunction("get_resources_for_workload")]
[Description("Gets the Resources under a workload provided the filtering conditions, and its corresponding Azores score.")]
[return: Description("The resources under a workload")]
public string get_resources_for_workload(
[Description("The Workload ID. If this filter is to be used, ALWAYS get the value of workload ID from function get_azores_workload_id_and_name_by_customer_id before proceeding")] string? workloadId = null,
[Description("The Customer ID of the customer. If this filter is to be used, ALWAYS get the value of customer ID from function get_customer_id_by_customer_name before proceeding")] string? customerId = null,
[Description("The Subscription ID in which the customer's resources are present")] string? subscriptionId = null,
[Description("The type of resource. Also known as type of product. For example - Storage Account, SQL DB, VM etc.")] string? resourceType = null,
[Description("The azure region in which the customer's resources are present")] string? region = null,
[Description("The boolean field which tells whether the resource is zone resilient or not.")] bool? isZoneResilient = null,
[Description("The number of records to retrieve. Also known as Top 'N'. Defaults to 5 if not provided")] int topN = 5
)
{
return $"2 resources found : SQLDB-T1 and SQLDB-V1"; // Dummy for Testing
}
[KernelFunction("get_azores_workload_id_and_name_by_customer_id")]
[Description("Gets the Azores Workloads for a customer provided the customer ID. All the workloads returned will have workload ID and workload name.")]
[return: Description("The workloads for a customer with their workload ID and workload name")]
public string get_azores_workload_id_and_name_by_customer_id([Description("The customer ID")] string customerId)
{
return string.Empty; // Dummy for Testing
}
[KernelFunction("get_azores_score_by_workload_name")]
[Description("Gets the Azores Workloads and their Azores score provided the workload name.")]
[return: Description("The azores workload id and its score")]
public string get_azores_score_by_workload_name([Description("The workload name")] string workloadName)
{
return string.Empty; //Dummy for Testing
}
}
class Program
{
static async Task Main(string[] args)
{
var resourceBuilder = ResourceBuilder
.CreateDefault()
.AddService("Contoso");
// Enable model diagnostics with sensitive data.
AppContext.SetSwitch("Microsoft.SemanticKernel.Experimental.GenAI.EnableOTelDiagnosticsSensitive", true);
using var traceProvider = Sdk.CreateTracerProviderBuilder()
.SetResourceBuilder(resourceBuilder)
.AddSource("Microsoft.SemanticKernel*")
.AddConsoleExporter()
.Build();
using var loggerFactory = LoggerFactory.Create(builder =>
{
// Add OpenTelemetry as a logging provider
builder.AddOpenTelemetry(options =>
{
options.SetResourceBuilder(resourceBuilder);
options.AddConsoleExporter();
// Format log messages. This is default to false.
options.IncludeFormattedMessage = true;
options.IncludeScopes = true;
});
builder.SetMinimumLevel(LogLevel.Information);
});
IKernelBuilder builder = Kernel.CreateBuilder();
builder.Services.AddSingleton(loggerFactory);
builder.AddAzureOpenAIChatCompletion(
deploymentName: "gpt-4o-mini",
endpoint: "--masked--",
apiKey: "--masked--"
);
builder.Plugins.AddFromType<CxObservePlugins>();
Kernel kernel = builder.Build();
IChatCompletionService chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
ChatHistory chatMessages = new("You are a friendly assistant inside a web application who likes to follow the rules. The web application contains data related to cloud company for its customers. You will complete required steps and request approval before taking any consequential actions. If the user doesn't provide enough information for you to complete a task, you will keep asking questions until you have enough information to complete the task. Please REFRAIN from answering any queries that are not part of your configured plugins. For example, you should not entertain questions like 'What is the weather like today?'. Only call the plugin when you are absolutely sure about the parameters. Accuracy is MOST IMPORTANT. ");
chatMessages.AddUserMessage("Can you check how many SQL resources for Contoso are non-zone resilient");
chatMessages.AddAssistantMessage("There are multiple entries for Contoso. Please specify which one you want to inquire about from the following list:\n\n1. Contoso Primary (Customer Id: 784852)\n2. Contoso Secondary (Customer Id: 44722548)\n\nPlease provide the specific name or ID you would like to use.");
chatMessages.AddUserMessage("784852");
chatMessages.AddAssistantMessage("The workloads for Contoso Primary (Customer Id: 784852) are:\n\n1. **Vibranium** (Workload Id: e4afc0ad-f671-428b-97b6-69e8a919b78b)\n2. **Shared Services - Networking** (Workload Id: a08f2a6f-53c2-48d9-8148-c76261a83505)\n3. **Adamantium** (Workload Id: 673ce7cb-5337-41e1-8cb0-5869769c6ac7)\n\nPlease specify which workload you want to check for non-zone resilient SQL resources.");
chatMessages.AddUserMessage("Check for All");
var result = await chatCompletionService.GetChatMessageContentsAsync(
chatMessages,
executionSettings: new()
{
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto()
},
kernel: kernel);
Console.WriteLine(result[^1].Content);
}
} |
ankiga-MSFT
changed the title
ankiga-MSFT
changed the title
ankiga-MSFT
changed the title
|
@ankiga-MSFT thanks for reporting this. @SergeyMenshykh can you take a look and see what is happening here. |
|
The model is sometimes returning function names with |
|
Hi @ankiga-MSFT, thanks for posting fully working C# code that reproduces the issue. It was very easy to get it running and reproduce the problem. The issue, though, was only reproducible with the gpt4o-mini, but the code worked perfectly fine with gpt4o. I played with the code a little bit and was able to reduce the function calling failure rate from 60% (6 out of 10 runs failed) to 0% using the gpt4o-mini model by just renaming the plugin functions - capitalizing each word in the name and removing underscores: class CxObservePlugins
{
[KernelFunction]
public string GetCustomerIdByCustomerName(string customerName)
{
return $"784852"; // Dummy for Testing
}
[KernelFunction]
public string GetWorkloadsForCustomerWithAzoresScore(
string? workloadId = null,
string? customerId = null,
double? azoresScoreLowerThreshold = null,
double? azoresScoreUpperThreshold = null,
string? operatorForAzoresThreshold = null,
int? zoneResilientResourceCount = null,
string? operatorForZoneResilientResourcesCount = null,
int? resourcesCount = null,
string? operatorForResourcesCount = null,
int topN = 10
)
{
return $"Vibranium (Workload Id: e4afc0ad-f671-428b-97b6-69e8a919b78b); Shared Services - Networking (Workload Id: a08f2a6f-53c2-48d9-8148-c76261a83505); Adamantium(Workload Id: 673ce7cb-5337-41e1-8cb0-5869769c6ac7";
}
[KernelFunction]
public string GetResourcesForWorkload(
string? workloadId = null,
string? customerId = null,
string? subscriptionId = null,
string? resourceType = null,
string? region = null,
bool? isZoneResilient = null,
int topN = 5
)
{
return $"2 resources found : SQLDB-T1 and SQLDB-V1"; // Dummy for Testing
}
[KernelFunction]
public string GetAzoresWorkloadIdAndNameByCustomerId(string customerId)
{
return string.Empty; // Dummy for Testing
}
[KernelFunction]
public string GetAzoresScoreByWorkloadName(string workloadName)
{
return string.Empty; //Dummy for Testing
}
}I was also able to decrease the number of consumed input tokens almost by half (from 1152 to 630) by removing descriptions for the functions, their parameters, and their return type, with no obvious regression; however, the scenario should be tested before doing so. I also noticed that you provided an alternative function name via the name parameter of the KernelFunction argument [KernelFunction("get_customer_id_by_customer_name")], which is the same as the function name defined in the plugin. It does not impact anything, but there are no benefits to doing it if both names are the same. Overall, my advice to have function calling work with gpt4o-mini would be to rename your functions if possible and to add descriptions only if your scenario requires them to safe a fair amount of tokens. CC: @yashworlikar |
|
I'm observing a similar failure:
|
|
Hi @maxgolov , thanks for sharing your experience. May I ask which model you use? How many functions does the impacted agent have? And would it be possible for you to distill your code into a console app that would reproduce the issue and share it with us for further investigation? |
|
@SergeyMenshykh Thanks for sharing the suggestions. Regarding the plugin function naming, I am following the recommendation mentioned in Semantic Kernel documentation, where it is mentioned to name the function using snake case. Quoting it - "Since most LLM have been trained with Python for function calling, its recommended to use snake case for function names and property names even if you're using the C# or Java SDK." (Reference). Also, do we know for sure, that changing the function nomenclature will fix the problem and we can rely on it in Production scenarios as well? I don't know whether this is Semantic Kernel SDK behavior or an issue with the gpt-4o-mini model, but any possibility of adding additional checks/validations at the SDK side? |
Thanks for pointing that out. We will revise it and update if the recommendation is no longer relevant.
I was able to increase the function calling success rate from 40% to 100% by just renaming the functions, so it definitely works with gpt4o-mini, but it may not necessarily work with other models. The best you can do at the moment is to test your scenarios thoroughly against the model you are going to use in production before going to production. You can use structural output for functions - .Net: Extend function calling model to support strict mode - to further improve function calling accuracy and parameter matching when it's implemented. The work has already started by one internal contributor. We are building a backlog of things that will improve the overall function calling experience in SK and plan to start working on it early next year. |
Describe the bug
Auto function is not able to call a registered function. It gives the error -
Invalid 'messages[6].tool_calls[0].function.name': string does not match pattern. Expected a string that matches the pattern '^[a-zA-Z0-9_-]+$'.'
This behavior is random, but more often than not, it is failing with the above error.
To Reproduce
Steps to reproduce the behavior:
Expected behavior
For the registered function in the plugin, the Auto Function calling should be able to call it successfully.
Platform
Additional context
Sharing the SDK Telemetry Logs below -
_Resource associated with LogRecord:
service.name: Contoso
service.instance.id: f32bad69-bfe5-4a9c-923c-2cb3905f1158
telemetry.sdk.name: opentelemetry
telemetry.sdk.language: dotnet
telemetry.sdk.version: 1.10.0
Activity.TraceId: 2317540eecaa7c4aca471a0776e179c0
Activity.SpanId: 0604e3a9d4898d85
Activity.TraceFlags: Recorded
Activity.DisplayName: chat.completions gpt-4o-mini
Activity.Kind: Client
Activity.StartTime: 2024-11-30T18:22:19.9198741Z
Activity.Duration: 00:00:03.0276928
Activity.Tags:
gen_ai.operation.name: chat.completions
gen_ai.system: openai
gen_ai.request.model: gpt-4o-mini
gen_ai.response.prompt_tokens: 1152
gen_ai.response.completion_tokens: 213
gen_ai.response.finish_reason: ToolCalls
gen_ai.response.id: chatcmpl-AZMOi8KAcoNwAJkHVTE2YuTUyFEYb
Activity.Events:
gen_ai.content.prompt [30-11-2024 18:22:19 +00:00]
gen_ai.prompt: [{"role": "system", "content": "You are a friendly assistant inside a web application who likes to follow the rules. The web application contains data related to cloud company for its customers. You will complete required steps and request approval before taking any consequential actions. If the user doesn\u0027t provide enough information for you to complete a task, you will keep asking questions until you have enough information to complete the task. Please REFRAIN from answering any queries that are not part of your configured plugins. For example, you should not entertain questions like \u0027What is the weather like today?\u0027. Only call the plugin when you are absolutely sure about the parameters. Accuracy is MOST IMPORTANT. "},
{"role": "user", "content": "Can you check how many SQL resources for Contoso are non-zone resilient"},
{"role": "assistant", "content": "There are multiple entries for Contoso. Please specify which one you want to inquire about from the following list:\n\n1. Contoso Primary (Customer Id: 784852)\n2. Contoso Secondary (Customer Id: 44722548)\n\nPlease provide the specific name or ID you would like to use."},
{"role": "user", "content": "784852"},
{"role": "assistant", "content": "The workloads for Contoso Primary (Customer Id: 784852) are:\n\n1. Vibranium (Workload Id: e4afc0ad-f671-428b-97b6-69e8a919b78b)\n2. Shared Services - Networking (Workload Id: a08f2a6f-53c2-48d9-8148-c76261a83505)\n3. Adamantium (Workload Id: 673ce7cb-5337-41e1-8cb0-5869769c6ac7)\n\nPlease specify which workload you want to check for non-zone resilient SQL resources."},
{"role": "user", "content": "Check for All"}]
gen_ai.content.completion [30-11-2024 18:22:22 +00:00]
gen_ai.completion: [{"role": "Assistant", "content": null, "tool_calls": [{"id": "call_dUXG33nmshqB7YJu6pdBOsPq", "function": {"arguments": {"workloadId":"e4afc0ad-f671-428b-97b6-69e8a919b78b","customerId":"784852","resourceType":"SQL DB","isZoneResilient":"False"}, "name": "CxObservePlugins.get_resources_for_workload"}, "type": "function"},
{"id": "call_1KsBX2AxzBRUfJ9ZDeLPKo9z", "function": {"arguments": {"workloadId":"a08f2a6f-53c2-48d9-8148-c76261a83505","customerId":"784852","resourceType":"SQL DB","isZoneResilient":"False"}, "name": "CxObservePlugins.get_resources_for_workload"}, "type": "function"},
{"id": "call_tXWWIurAH2xgnO8W1yp91MAE", "function": {"arguments": {"workloadId":"673ce7cb-5337-41e1-8cb0-5869769c6ac7","customerId":"784852","resourceType":"SQL DB","isZoneResilient":"False"}, "name": "CxObservePlugins.get_resources_for_workload"}, "type": "function"}]}]
Instrumentation scope (ActivitySource):
Name: Microsoft.SemanticKernel.Diagnostics
Resource associated with Activity:
service.name: Contoso
service.instance.id: f32bad69-bfe5-4a9c-923c-2cb3905f1158
telemetry.sdk.name: opentelemetry
telemetry.sdk.language: dotnet
telemetry.sdk.version: 1.10.0
Activity.TraceId: b09c4063c5af28d20183fa1fd521fe31
Activity.SpanId: c385295f604cb7a9
Activity.TraceFlags: Recorded
Activity.DisplayName: chat.completions gpt-4o-mini
Activity.Kind: Client
Activity.StartTime: 2024-11-30T18:22:22.9611424Z
Activity.Duration: 00:00:00.3450625
Activity.Tags:
gen_ai.operation.name: chat.completions
gen_ai.system: openai
gen_ai.request.model: gpt-4o-mini
error.type: Microsoft.SemanticKernel.HttpOperationException
StatusCode: Error
Activity.StatusDescription: HTTP 400 (invalid_request_error: invalid_value)
Parameter: messages[6].tool_calls[0].function.name
Invalid 'messages[6].tool_calls[0].function.name': string does not match pattern. Expected a string that matches the pattern '^[a-zA-Z0-9_-]+$'.
Activity.Events:
gen_ai.content.prompt [30-11-2024 18:22:22 +00:00]
gen_ai.prompt: [{"role": "system", "content": "You are a friendly assistant inside a web application who likes to follow the rules. The web application contains data related to cloud company for its customers. You will complete required steps and request approval before taking any consequential actions. If the user doesn\u0027t provide enough information for you to complete a task, you will keep asking questions until you have enough information to complete the task. Please REFRAIN from answering any queries that are not part of your configured plugins. For example, you should not entertain questions like \u0027What is the weather like today?\u0027. Only call the plugin when you are absolutely sure about the parameters. Accuracy is MOST IMPORTANT. "},
{"role": "user", "content": "Can you check how many SQL resources for Contoso are non-zone resilient"},
{"role": "assistant", "content": "There are multiple entries for Contoso. Please specify which one you want to inquire about from the following list:\n\n1. Contoso Primary (Customer Id: 784852)\n2. Contoso Secondary (Customer Id: 44722548)\n\nPlease provide the specific name or ID you would like to use."},
{"role": "user", "content": "784852"},
{"role": "assistant", "content": "The workloads for Contoso Primary (Customer Id: 784852) are:\n\n1. Vibranium (Workload Id: e4afc0ad-f671-428b-97b6-69e8a919b78b)\n2. Shared Services - Networking (Workload Id: a08f2a6f-53c2-48d9-8148-c76261a83505)\n3. Adamantium (Workload Id: 673ce7cb-5337-41e1-8cb0-5869769c6ac7)\n\nPlease specify which workload you want to check for non-zone resilient SQL resources."},
{"role": "user", "content": "Check for All"},
{"role": "Assistant", "content": null, "tool_calls": [{"id": "call_dUXG33nmshqB7YJu6pdBOsPq", "function": {"arguments": {"workloadId":"e4afc0ad-f671-428b-97b6-69e8a919b78b","customerId":"784852","resourceType":"SQL DB","isZoneResilient":"False"}, "name": "CxObservePlugins.get_resources_for_workload"}, "type": "function"},
{"id": "call_1KsBX2AxzBRUfJ9ZDeLPKo9z", "function": {"arguments": {"workloadId":"a08f2a6f-53c2-48d9-8148-c76261a83505","customerId":"784852","resourceType":"SQL DB","isZoneResilient":"False"}, "name": "CxObservePlugins.get_resources_for_workload"}, "type": "function"},
{"id": "call_tXWWIurAH2xgnO8W1yp91MAE", "function": {"arguments": {"workloadId":"673ce7cb-5337-41e1-8cb0-5869769c6ac7","customerId":"784852","resourceType":"SQL DB","isZoneResilient":"False"}, "name": "CxObservePlugins.get_resources_for_workload"}, "type": "function"}]},
{"role": "tool", "content": "Error: Function call request for a function that wasn\u0027t defined."},
{"role": "tool", "content": "Error: Function call request for a function that wasn\u0027t defined."},
{"role": "tool", "content": "Error: Function call request for a function that wasn\u0027t defined."}]
Instrumentation scope (ActivitySource):
Name: Microsoft.SemanticKernel.Diagnostics
Resource associated with Activity:
service.name: Contoso
service.instance.id: f32bad69-bfe5-4a9c-923c-2cb3905f1158
telemetry.sdk.name: opentelemetry
telemetry.sdk.language: dotnet
telemetry.sdk.version: 1.10.0_
The text was updated successfully, but these errors were encountered: